YAO/YAO-optimizer
1
0
Fork 0
mirror of https://github.com/newnius/YAO-optimizer.git synced 2025-06-07 15:11:56 +00:00
Code Issues Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source 
# Conflicts:
#	main.py
This commit is contained in:
yexiaoqi 2020-05-02 13:07:26 +08:00
commit 1fd4ab4052
6 changed files with 386 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
.idea/workspace.xml
View File

@ -1,9 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="0aedafd8-e57e-462a-beda-65af0b91f3df" name="Default Changelist" comment=""> <list default="true" id="0aedafd8-e57e-462a-beda-65af0b91f3df" name="Default Changelist" comment="" />
<change beforePath="$PROJECT_DIR$/serve.py" beforeDir="false" afterPath="$PROJECT_DIR$/serve.py" afterDir="false" />
</list>
<ignored path="$PROJECT_DIR$/out/" /> <ignored path="$PROJECT_DIR$/out/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" /> <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" /> <option name="SHOW_DIALOG" value="false" />
@ -15,8 +13,9 @@
<session id="570274097"> <session id="570274097">
<usages-collector id="statistics.lifecycle.project"> <usages-collector id="statistics.lifecycle.project">
<counts> <counts>
<entry key="project.open.time.1" value="1" />
<entry key="project.open.time.3" value="1" /> <entry key="project.open.time.3" value="1" />
<entry key="project.opened" value="1" /> <entry key="project.opened" value="2" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.open"> <usages-collector id="statistics.file.extensions.open">
@ -24,43 +23,49 @@
<entry key="Dockerfile" value="1" /> <entry key="Dockerfile" value="1" />
<entry key="csv" value="2" /> <entry key="csv" value="2" />
<entry key="iml" value="1" /> <entry key="iml" value="1" />
<entry key="md" value="1" /> <entry key="md" value="2" />
<entry key="png" value="7" /> <entry key="png" value="7" />
<entry key="py" value="6" /> <entry key="py" value="7" />
<entry key="sh" value="2" />
<entry key="txt" value="2" /> <entry key="txt" value="2" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.open"> <usages-collector id="statistics.file.types.open">
<counts> <counts>
<entry key="Bash" value="2" />
<entry key="Dockerfile" value="1" /> <entry key="Dockerfile" value="1" />
<entry key="IDEA_MODULE" value="1" /> <entry key="IDEA_MODULE" value="1" />
<entry key="Image" value="7" /> <entry key="Image" value="7" />
<entry key="Markdown" value="1" /> <entry key="Markdown" value="2" />
<entry key="PLAIN_TEXT" value="4" /> <entry key="PLAIN_TEXT" value="4" />
<entry key="Python" value="6" /> <entry key="Python" value="7" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.extensions.edit"> <usages-collector id="statistics.file.extensions.edit">
<counts> <counts>
<entry key="Dockerfile" value="16" /> <entry key="Dockerfile" value="81" />
<entry key="py" value="1357" /> <entry key="md" value="104" />
<entry key="py" value="2825" />
<entry key="sh" value="5" />
</counts> </counts>
</usages-collector> </usages-collector>
<usages-collector id="statistics.file.types.edit"> <usages-collector id="statistics.file.types.edit">
<counts> <counts>
<entry key="Dockerfile" value="16" /> <entry key="Bash" value="5" />
<entry key="Python" value="1357" /> <entry key="Dockerfile" value="81" />
<entry key="Markdown" value="104" />
<entry key="Python" value="2825" />
</counts> </counts>
</usages-collector> </usages-collector>
</session> </session>
</component> </component>
<component name="FileEditorManager"> <component name="FileEditorManager">
<leaf> <leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/main.py"> <entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2160"> <state relative-caret-position="176">
<caret line="144" column="4" selection-start-line="144" selection-start-column="4" selection-end-line="144" selection-end-column="30" /> <caret line="199" column="32" lean-forward="true" selection-start-line="199" selection-start-column="32" selection-end-line="199" selection-end-column="32" />
<folding> <folding>
<element signature="e#0#19#0" expanded="true" /> <element signature="e#0#19#0" expanded="true" />
</folding> </folding>
@ -69,41 +74,56 @@
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/data/stock_data.csv"> <entry file="file://$PROJECT_DIR$/serve.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="60"> <state relative-caret-position="-1056">
<caret line="4" column="37" selection-start-line="4" selection-start-column="37" selection-end-line="4" selection-end-column="37" /> <caret line="43" column="16" lean-forward="true" selection-start-line="43" selection-start-column="16" selection-end-line="43" selection-end-column="16" />
<folding>
<element signature="e#18#46#0" expanded="true" />
<marker date="1588357497851" expanded="true" signature="5576:5578" ph="..." />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="true"> <file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/serve.py"> <entry file="file://$PROJECT_DIR$/train.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="260"> <state relative-caret-position="1580">
<caret line="53" column="28" selection-start-line="53" selection-start-column="28" selection-end-line="53" selection-end-column="28" /> <caret line="110" column="61" lean-forward="true" selection-start-line="110" selection-start-column="61" selection-end-line="110" selection-end-column="62" />
<folding> <folding>
<element signature="e#18#46#0" expanded="true" /> <element signature="e#0#28#0" expanded="true" />
<marker date="1588180249558" expanded="true" signature="5697:5971" ph="..." />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="SPLIT">
<first_editor relative-caret-position="105">
<caret line="7" column="6" selection-start-line="7" selection-start-column="6" selection-end-line="7" selection-end-column="6" />
</first_editor>
<second_editor />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/Dockerfile"> <entry file="file://$PROJECT_DIR$/Dockerfile">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180"> <state relative-caret-position="180">
<caret line="12" column="24" selection-start-line="12" selection-start-column="24" selection-end-line="12" selection-end-column="24" /> <caret line="12" column="58" selection-start-line="12" selection-start-column="58" selection-end-line="12" selection-end-column="58" />
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/requirements.txt"> <entry file="file://$PROJECT_DIR$/bootstrap.sh">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120"> <state relative-caret-position="30">
<caret line="8" lean-forward="true" selection-start-line="8" selection-end-line="8" /> <caret line="2" column="22" lean-forward="true" selection-start-line="2" selection-start-column="22" selection-end-line="2" selection-end-column="22" />
</state> </state>
</provider> </provider>
</entry> </entry>
@ -111,8 +131,8 @@
<file pinned="false" current-in-tab="false"> <file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/model_tensorflow.py"> <entry file="file://$PROJECT_DIR$/model_tensorflow.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="358"> <state relative-caret-position="157">
<caret line="59" column="45" selection-start-line="59" selection-start-column="45" selection-end-line="59" selection-end-column="45" /> <caret line="50" column="56" selection-start-line="50" selection-start-column="56" selection-end-line="50" selection-end-column="56" />
<folding> <folding>
<element signature="e#0#23#0" expanded="true" /> <element signature="e#0#23#0" expanded="true" />
</folding> </folding>
@ -134,6 +154,16 @@
<find>lstm</find> <find>lstm</find>
<find>used</find> <find>used</find>
<find>mean</find> <find>mean</find>
<find>std</find>
<find>usecol</find>
<find>valid_data_rate</find>
<find>batch_size</find>
<find>predict</find>
<find>train_num</find>
<find>label_in_feature_columns</find>
<find>epoch</find>
<find>timeseries_to_supervised</find>
<find>batch_index</find>
</findStrings> </findStrings>
</component> </component>
<component name="Git.Settings"> <component name="Git.Settings">
@ -145,10 +175,13 @@
<option value="$PROJECT_DIR$/test.py" /> <option value="$PROJECT_DIR$/test.py" />
<option value="$PROJECT_DIR$/model/model_tensorflow.py" /> <option value="$PROJECT_DIR$/model/model_tensorflow.py" />
<option value="$PROJECT_DIR$/requirements.txt" /> <option value="$PROJECT_DIR$/requirements.txt" />
<option value="$PROJECT_DIR$/main.py" /> <option value="$PROJECT_DIR$/bootstrap.sh" />
<option value="$PROJECT_DIR$/Dockerfile" /> <option value="$PROJECT_DIR$/README.md" />
<option value="$PROJECT_DIR$/model_tensorflow.py" /> <option value="$PROJECT_DIR$/model_tensorflow.py" />
<option value="$PROJECT_DIR$/Dockerfile" />
<option value="$PROJECT_DIR$/serve.py" /> <option value="$PROJECT_DIR$/serve.py" />
<option value="$PROJECT_DIR$/main.py" />
<option value="$PROJECT_DIR$/train.py" />
</list> </list>
</option> </option>
</component> </component>
@ -167,8 +200,6 @@
<foldersAlwaysOnTop value="true" /> <foldersAlwaysOnTop value="true" />
</navigator> </navigator>
<panes> <panes>
<pane id="AndroidView" />
<pane id="PackagesPane" />
<pane id="ProjectPane"> <pane id="ProjectPane">
<subPane> <subPane>
<expand> <expand>
@ -185,13 +216,15 @@
<select /> <select />
</subPane> </subPane>
</pane> </pane>
<pane id="AndroidView" />
<pane id="PackagesPane" />
<pane id="Scope" /> <pane id="Scope" />
</panes> </panes>
</component> </component>
<component name="PropertiesComponent"> <component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" /> <property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" /> <property name="aspect.path.notification.shown" value="true" />
<property name="com.android.tools.idea.instantapp.provision.ProvisionBeforeRunTaskProvider.myTimeStamp" value="1588180252829" /> <property name="com.android.tools.idea.instantapp.provision.ProvisionBeforeRunTaskProvider.myTimeStamp" value="1588395483884" />
<property name="go.gopath.indexing.explicitly.defined" value="true" /> <property name="go.gopath.indexing.explicitly.defined" value="true" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" /> <property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" /> <property name="nodejs_npm_path_reset_for_default_project" value="true" />
@ -201,13 +234,13 @@
<property name="settings.editor.selected.configurable" value="http.proxy" /> <property name="settings.editor.selected.configurable" value="http.proxy" />
</component> </component>
<component name="RecentsManager"> <component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$" />
</key>
<key name="MoveFile.RECENT_KEYS"> <key name="MoveFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$" /> <recent name="$PROJECT_DIR$" />
<recent name="$PROJECT_DIR$/model" /> <recent name="$PROJECT_DIR$/model" />
</key> </key>
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$" />
</key>
</component> </component>
<component name="RunDashboard"> <component name="RunDashboard">
<option name="ruleStates"> <option name="ruleStates">
@ -232,21 +265,23 @@
<option name="presentableId" value="Default" /> <option name="presentableId" value="Default" />
<updated>1588152877746</updated> <updated>1588152877746</updated>
<workItem from="1588152880522" duration="16973000" /> <workItem from="1588152880522" duration="16973000" />
<workItem from="1588319878551" duration="26780000" />
</task> </task>
<servers /> <servers />
</component> </component>
<component name="TimeTrackingManager"> <component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="16973000" /> <option name="totallyTimeSpent" value="43753000" />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="0" y="0" width="1280" height="800" extended-state="0" /> <frame x="0" y="0" width="1280" height="800" extended-state="0" />
<editor active="true" />
<layout> <layout>
<window_info id="Designer" order="0" /> <window_info id="Designer" order="0" />
<window_info id="UI Designer" order="1" /> <window_info id="UI Designer" order="1" />
<window_info id="Capture Tool" order="2" /> <window_info id="Capture Tool" order="2" />
<window_info id="Favorites" order="3" side_tool="true" /> <window_info id="Favorites" order="3" side_tool="true" />
<window_info id="Image Layers" order="4" /> <window_info id="Image Layers" order="4" />
<window_info content_ui="combo" id="Project" order="5" sideWeight="0.49898168" visible="true" weight="0.24959612" /> <window_info content_ui="combo" id="Project" order="5" sideWeight="0.49898168" visible="true" weight="0.2536349" />
<window_info id="Structure" order="6" sideWeight="0.50101835" side_tool="true" weight="0.24959612" /> <window_info id="Structure" order="6" sideWeight="0.50101835" side_tool="true" weight="0.24959612" />
<window_info anchor="bottom" id="Version Control" order="0" /> <window_info anchor="bottom" id="Version Control" order="0" />
<window_info active="true" anchor="bottom" id="Terminal" order="1" visible="true" weight="0.32739726" /> <window_info active="true" anchor="bottom" id="Terminal" order="1" visible="true" weight="0.32739726" />
@ -282,59 +317,25 @@
<entry file="file://$PROJECT_DIR$/yao-optimizer.iml"> <entry file="file://$PROJECT_DIR$/yao-optimizer.iml">
<provider selected="true" editor-type-id="text-editor" /> <provider selected="true" editor-type-id="text-editor" />
</entry> </entry>
<entry file="file://$PROJECT_DIR$/test.py"> <entry file="file://$PROJECT_DIR$/test.py" />
<provider selected="true" editor-type-id="text-editor"> <entry file="file://$PROJECT_DIR$/figure/continue_predict_high_with_pytorch.png" />
<state relative-caret-position="150"> <entry file="file://$PROJECT_DIR$/figure/continue_predict_low_with_pytorch.png" />
<caret line="52" column="30" selection-start-line="52" selection-start-column="30" selection-end-line="52" selection-end-column="30" /> <entry file="file://$PROJECT_DIR$/figure/predict_high_with_keras.png" />
<folding> <entry file="file://$PROJECT_DIR$/figure/predict_low_with_tensorflow.png" />
<element signature="e#0#19#0" expanded="true" /> <entry file="file://$PROJECT_DIR$/figure/predict_low_with_keras.png" />
</folding> <entry file="file://$PROJECT_DIR$/figure/predict_high_with_tensorflow.png" />
</state> <entry file="file://$PROJECT_DIR$/figure/predict_high_with_pytorch.png" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/figure/continue_predict_high_with_pytorch.png">
<provider selected="true" editor-type-id="images" />
</entry>
<entry file="file://$PROJECT_DIR$/figure/continue_predict_low_with_pytorch.png">
<provider selected="true" editor-type-id="images" />
</entry>
<entry file="file://$PROJECT_DIR$/figure/predict_high_with_keras.png">
<provider selected="true" editor-type-id="images" />
</entry>
<entry file="file://$PROJECT_DIR$/figure/predict_low_with_tensorflow.png">
<provider selected="true" editor-type-id="images" />
</entry>
<entry file="file://$PROJECT_DIR$/figure/predict_low_with_keras.png">
<provider selected="true" editor-type-id="images" />
</entry>
<entry file="file://$PROJECT_DIR$/figure/predict_high_with_tensorflow.png">
<provider selected="true" editor-type-id="images" />
</entry>
<entry file="file://$PROJECT_DIR$/figure/predict_high_with_pytorch.png">
<provider selected="true" editor-type-id="images" />
</entry>
<entry file="file://$PROJECT_DIR$/Readme.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="SPLIT">
<first_editor />
<second_editor />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/requirements.txt"> <entry file="file://$PROJECT_DIR$/requirements.txt">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="120"> <state relative-caret-position="120">
<caret line="8" lean-forward="true" selection-start-line="8" selection-end-line="8" /> <caret line="8" selection-start-line="8" selection-end-line="8" />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/main.py"> <entry file="file://$PROJECT_DIR$/bootstrap.sh">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="2160"> <state relative-caret-position="30">
<caret line="144" column="4" selection-start-line="144" selection-start-column="4" selection-end-line="144" selection-end-column="30" /> <caret line="2" column="22" lean-forward="true" selection-start-line="2" selection-start-column="22" selection-end-line="2" selection-end-column="22" />
<folding>
<element signature="e#0#19#0" expanded="true" />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
@ -348,27 +349,57 @@
<entry file="file://$PROJECT_DIR$/Dockerfile"> <entry file="file://$PROJECT_DIR$/Dockerfile">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180"> <state relative-caret-position="180">
<caret line="12" column="24" selection-start-line="12" selection-start-column="24" selection-end-line="12" selection-end-column="24" /> <caret line="12" column="58" selection-start-line="12" selection-start-column="58" selection-end-line="12" selection-end-column="58" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/README.md">
<provider selected="true" editor-type-id="split-provider[text-editor;markdown-preview-editor]">
<state split_layout="SPLIT">
<first_editor relative-caret-position="105">
<caret line="7" column="6" selection-start-line="7" selection-start-column="6" selection-end-line="7" selection-end-column="6" />
</first_editor>
<second_editor />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/serve.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-1056">
<caret line="43" column="16" lean-forward="true" selection-start-line="43" selection-start-column="16" selection-end-line="43" selection-end-column="16" />
<folding>
<element signature="e#18#46#0" expanded="true" />
<marker date="1588357497851" expanded="true" signature="5576:5578" ph="..." />
</folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/model_tensorflow.py"> <entry file="file://$PROJECT_DIR$/model_tensorflow.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="358"> <state relative-caret-position="157">
<caret line="59" column="45" selection-start-line="59" selection-start-column="45" selection-end-line="59" selection-end-column="45" /> <caret line="50" column="56" selection-start-line="50" selection-start-column="56" selection-end-line="50" selection-end-column="56" />
<folding> <folding>
<element signature="e#0#23#0" expanded="true" /> <element signature="e#0#23#0" expanded="true" />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/serve.py"> <entry file="file://$PROJECT_DIR$/main.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="260"> <state relative-caret-position="176">
<caret line="53" column="28" selection-start-line="53" selection-start-column="28" selection-end-line="53" selection-end-column="28" /> <caret line="199" column="32" lean-forward="true" selection-start-line="199" selection-start-column="32" selection-end-line="199" selection-end-column="32" />
<folding> <folding>
<element signature="e#18#46#0" expanded="true" /> <element signature="e#0#19#0" expanded="true" />
<marker date="1588180249558" expanded="true" signature="5697:5971" ph="..." /> </folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/train.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1580">
<caret line="110" column="61" lean-forward="true" selection-start-line="110" selection-start-column="61" selection-end-line="110" selection-end-column="62" />
<folding>
<element signature="e#0#28#0" expanded="true" />
</folding> </folding>
</state> </state>
</provider> </provider>

6
Dockerfile
View File

@ -10,11 +10,13 @@ RUN apt update && \
apt install -y git vim httpie && \ apt install -y git vim httpie && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
RUN pip3 install pandas sklearn tensorflow-gpu==1.14 RUN pip3 install pandas sklearn tensorflow-gpu==1.14 keras
ADD bootstrap.sh /etc/bootstrap.sh ADD bootstrap.sh /etc/bootstrap.sh
ADD agent.py /root/agent.py RUN mkdir /root/data/
ADD serve.py /root/serve.py
ADD model_tensorflow.py /root/model_tensorflow.py
WORKDIR /root WORKDIR /root

11
README.md Normal file
View File

@ -0,0 +1,11 @@
## Feed
/feed?job=1&model=2&time=3&utilGPU=4&utilCPU=5&pre=1&main=2&post=3
## train
/train
## predict
/predict?job=1&model=2&time=3&utilGPU=4&utilCPU=5

3
bootstrap.sh Executable file
View File

@ -0,0 +1,3 @@
#!/usr/bin/env bash
python3 /root/serve.py

136
serve.py
View File

@ -14,8 +14,8 @@ import csv
class Config: class Config:
feature_columns = list(range(0, 6)) feature_columns = list(range(0, 2))
label_columns = [3, 4, 5] label_columns = [1]
feature_and_label_columns = feature_columns + label_columns feature_and_label_columns = feature_columns + label_columns
label_in_feature_columns = (lambda x, y: [x.index(i) for i in y])(feature_columns, label_columns) label_in_feature_columns = (lambda x, y: [x.index(i) for i in y])(feature_columns, label_columns)
@ -34,7 +34,8 @@ class Config:
add_train = False add_train = False
shuffle_train_data = True shuffle_train_data = True
train_data_rate = 0.95 # train_data_rate = 0.95 #comment yqy
train_data_rate = 1 # add yqy
valid_data_rate = 0.15 valid_data_rate = 0.15
batch_size = 64 batch_size = 64
@ -50,10 +51,10 @@ class Config:
batch_size = 1 batch_size = 1
continue_flag = "continue_" continue_flag = "continue_"
train_data_path = "./data.csv" train_data_path = "./data/data.csv"
test_data_path = "./test.csv"
model_save_path = "./checkpoint/" model_save_path = "./checkpoint/"
figure_save_path = "./figure/" figure_save_path = "./figure/"
do_figure_save = False do_figure_save = False
if not os.path.exists(model_save_path): if not os.path.exists(model_save_path):
os.mkdir(model_save_path) os.mkdir(model_save_path)
@ -74,17 +75,14 @@ class Data:
self.train_num = int(self.data_num * self.config.train_data_rate) self.train_num = int(self.data_num * self.config.train_data_rate)
self.mean = np.mean(self.data, axis=0) self.mean = np.mean(self.data, axis=0)
self.std = np.std(self.data, axis=0) + 0.0001
self.std = np.std(self.data, axis=0)
self.norm_data = (self.data - self.mean) / self.std self.norm_data = (self.data - self.mean) / self.std
self.start_num_in_test = 0 self.start_num_in_test = 0
def read_data(self): def read_data(self):
init_data = pd.read_csv( init_data = pd.read_csv(self.config.train_data_path,
self.config.train_data_path, usecols=self.config.feature_and_label_columns)
usecols=self.config.feature_and_label_columns
)
return init_data.values, init_data.columns.tolist() return init_data.values, init_data.columns.tolist()
def get_train_and_valid_data(self): def get_train_and_valid_data(self):
@ -112,22 +110,6 @@ class Data:
return train_x, valid_x, train_y, valid_y return train_x, valid_x, train_y, valid_y
def get_test_data(self, return_label_data=False): def get_test_data(self, return_label_data=False):
init_data = pd.read_csv(
self.config.test_data_path,
usecols=self.config.feature_and_label_columns
)
norm_data = (init_data - self.mean) / self.std
feature_data = norm_data[:]
time_step_size = feature_data.shape[0] // self.config.time_step
test_x = [feature_data[0 + i * self.config.time_step: 0 + (
i + 1) * self.config.time_step]
for i in range(time_step_size)]
if return_label_data:
label_data = norm_data[0:, self.config.label_in_feature_columns]
return np.array(test_x), label_data
return np.array(test_x)
feature_data = self.norm_data[self.train_num:] feature_data = self.norm_data[self.train_num:]
self.start_num_in_test = feature_data.shape[0] % self.config.time_step self.start_num_in_test = feature_data.shape[0] % self.config.time_step
time_step_size = feature_data.shape[0] // self.config.time_step time_step_size = feature_data.shape[0] // self.config.time_step
@ -140,31 +122,50 @@ class Data:
return np.array(test_x), label_data return np.array(test_x), label_data
return np.array(test_x) return np.array(test_x)
# add yqy
def get_test_data_yqy(self, test_data_yqy=None):
if test_data_yqy is None:
test_data_yqy = []
# test_data_yqy=test_data_yqy[1:21]
feature_data = (test_data_yqy - self.mean) / self.std
test_x = [feature_data]
return np.array(test_x)
def draw(config, origin_data, predict_norm_data):
label_norm_data = origin_data.norm_data[origin_data.train_num + origin_data.start_num_in_test:, # add end
config.label_in_feature_columns]
def draw_yqy(config2, origin_data, predict_norm_data, mean_yqy, std_yqy):
label_norm_data = (origin_data - mean_yqy) / std_yqy
assert label_norm_data.shape[0] == predict_norm_data.shape[ assert label_norm_data.shape[0] == predict_norm_data.shape[
0], "The element number in origin and predicted data is different" 0], "The element number in origin and predicted data is different"
label_name = [origin_data.data_column_name[i] for i in config.label_in_feature_columns] print("dsa")
label_column_num = len(config.label_columns) # label_norm_data=label_norm_data[:,1]
label_name = 'high'
label_column_num = 3
loss = np.mean((label_norm_data[config.predict_day:] - predict_norm_data[:-config.predict_day]) ** 2, axis=0) loss = \
np.mean((label_norm_data[config.predict_day:, 1:2] - predict_norm_data[:-config.predict_day]) ** 2, axis=0)
print("The mean squared error of stock {} is ".format(label_name), loss) print("The mean squared error of stock {} is ".format(label_name), loss)
label_X = range(origin_data.data_num - origin_data.train_num - origin_data.start_num_in_test) # label_X = range(origin_data.data_num - origin_data.train_num - origin_data.start_num_in_test)
predict_X = [x + config.predict_day for x in label_X] # predict_X = [x + config.predict_day for x in label_X]
label_data = label_norm_data * origin_data.std[config.label_in_feature_columns] + \ print("2")
origin_data.mean[config.label_in_feature_columns]
predict_data = predict_norm_data * origin_data.std[config.label_in_feature_columns] + \
origin_data.mean[config.label_in_feature_columns]
print(label_norm_data[:, 1:2])
label_data = label_norm_data[:, 1:2] * std_yqy[1:2] + mean_yqy[1:2]
print(label_data) print(label_data)
print(predict_norm_data)
predict_data = predict_norm_data * std_yqy[config.label_in_feature_columns] + mean_yqy[
config.label_in_feature_columns]
print(predict_data) print(predict_data)
print(label_data[:, -1])
print(predict_data[:, -1])
PORT_NUMBER = 8080 PORT_NUMBER = 8080
lock = Lock() lock = Lock()
@ -179,6 +180,11 @@ def train_models():
train_X, valid_X, train_Y, valid_Y = data_gainer.get_train_and_valid_data() train_X, valid_X, train_Y, valid_Y = data_gainer.get_train_and_valid_data()
print(train_X, valid_X, train_Y, valid_Y) print(train_X, valid_X, train_Y, valid_Y)
print(train_X.shape[0])
if train_X.shape[0] < 500:
config.batch_size = 32
if train_X.shape[0] < 200:
config.batch_size = 16
train(config, train_X, train_Y, valid_X, valid_Y) train(config, train_X, train_Y, valid_X, valid_Y)
@ -199,15 +205,39 @@ class MyHandler(BaseHTTPRequestHandler):
elif req.path == "/predict": elif req.path == "/predict":
try: try:
job = query.get('job')[0] data = {
gpu_model = query.get('gpu_model')[0] 'job': query.get('job')[0],
time = query.get('time')[0] 'model': query.get('model')[0],
'time': query.get('time')[0],
'utilGPU': query.get('utilGPU')[0],
'utilCPU': query.get('utilCPU')[0],
'pre': 0,
'main': 0,
'post': 0
}
data = {
'seq': query.get('job')[0],
'value': query.get('model')[0],
}
with open(config.train_data_path, 'r') as f:
df = pd.read_csv(config.train_data_path, usecols=['seq', 'value'])
df = df.tail(config.time_step - 1)
df = df.append(data, ignore_index=True)
df.to_csv('./data/test_data.csv', index=False)
np.random.seed(config.random_seed)
data_gainer = Data(config) data_gainer = Data(config)
test_X, test_Y = data_gainer.get_test_data(return_label_data=True) test_data_yqy = pd.read_csv("./data/test_data.csv", usecols=list(range(0, 2)))
print(test_X, test_Y) test_data_values = test_data_yqy.values[:]
test_X = data_gainer.get_test_data_yqy(test_data_values)
pred_result = predict(config, test_X) pred_result = predict(config, test_X)
print(pred_result)
draw(config, data_gainer, pred_result) mean = Data(config).mean
std = Data(config).std
draw_yqy(config, test_data_values, pred_result, mean, std)
msg = {'code': 1, 'error': "container not exist"} msg = {'code': 1, 'error': "container not exist"}
except Exception as e: except Exception as e:
msg = {'code': 2, 'error': str(e)} msg = {'code': 2, 'error': str(e)}
@ -221,15 +251,22 @@ class MyHandler(BaseHTTPRequestHandler):
job = query.get('job')[0] job = query.get('job')[0]
model = query.get('model')[0] model = query.get('model')[0]
time = query.get('time')[0] time = query.get('time')[0]
utilGPU = query.get('utilGPU')[0]
utilCPU = query.get('utilCPU')[0]
pre = query.get('pre')[0] pre = query.get('pre')[0]
main = query.get('main')[0] main = query.get('main')[0]
post = query.get('post')[0] post = query.get('post')[0]
seq = query.get('seq')[0]
value = query.get('value')[0]
with open(config.train_data_path, 'a+', newline='') as csvfile: with open(config.train_data_path, 'a+', newline='') as csvfile:
spamwriter = csv.writer( spamwriter = csv.writer(
csvfile, delimiter=',', csvfile, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL quotechar='|', quoting=csv.QUOTE_MINIMAL
) )
spamwriter.writerow([job, model, time, pre, main, post]) # spamwriter.writerow([job, model, time, utilGPU, utilCPU, pre, main, post])
spamwriter.writerow([seq, value])
msg = {'code': 1, 'error': "container not exist"} msg = {'code': 1, 'error': "container not exist"}
except Exception as e: except Exception as e:
msg = {'code': 2, 'error': str(e)} msg = {'code': 2, 'error': str(e)}
@ -293,7 +330,8 @@ if __name__ == '__main__':
csvfile, delimiter=',', csvfile, delimiter=',',
quotechar='|', quoting=csv.QUOTE_MINIMAL quotechar='|', quoting=csv.QUOTE_MINIMAL
) )
spamwriter.writerow(["Job", "Model", "Time", "Pre", "Main", "Post"]) #spamwriter.writerow(["job", "model", "time", "utilGPU", "utilCPU", "pre", "main", "post"])
spamwriter.writerow(["seq", "value"])
# Wait forever for incoming http requests # Wait forever for incoming http requests
server.serve_forever() server.serve_forever()

157
train.py Normal file
View File

@ -0,0 +1,157 @@
from pandas import DataFrame
from pandas import Series
from pandas import concat
from pandas import read_csv
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from math import sqrt
import numpy
# frame a sequence as a supervised learning problem
def timeseries_to_supervised(data, lag=1):
df = DataFrame(data)
columns = [df.shift(i) for i in range(1, lag + 1)]
columns.append(df)
df = concat(columns, axis=1)
df = df.drop(0)
return df
# create a differenced series
def difference(dataset, interval=1):
diff = list()
for i in range(interval, len(dataset)):
value = dataset[i] - dataset[i - interval]
diff.append(value)
return Series(diff)
# invert differenced value
def inverse_difference(history, yhat, interval=1):
return yhat + history[-interval]
# scale train and test data to [-1, 1]
def scale(train, test):
# fit scaler
scaler = MinMaxScaler(feature_range=(-1, 1))
scaler = scaler.fit(train)
# transform train
train = train.reshape(train.shape[0], train.shape[1])
train_scaled = scaler.transform(train)
# transform test
test = test.reshape(test.shape[0], test.shape[1])
test_scaled = scaler.transform(test)
return scaler, train_scaled, test_scaled
# inverse scaling for a forecasted value
def invert_scale(scaler, X, yhat):
new_row = [x for x in X] + [yhat]
array = numpy.array(new_row)
array = array.reshape(1, len(array))
inverted = scaler.inverse_transform(array)
return inverted[0, -1]
# fit an LSTM network to training data
def fit_lstm(train, batch_size2, nb_epoch, neurons):
print(train)
X, y = train[:, 0:-1], train[:, -1]
X = X.reshape(X.shape[0], 1, X.shape[1])
print(X, y)
model = Sequential()
model.add(LSTM(neurons, batch_input_shape=(batch_size2, X.shape[1], X.shape[2]), stateful=True))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
for i in range(nb_epoch):
print("Epoch {}/{}".format(i, nb_epoch))
model.fit(X, y, epochs=1, batch_size=batch_size2, verbose=0, shuffle=False)
#loss, accuracy = model.evaluate(X, y)
#print(loss, accuracy)
model.reset_states()
return model
# make a one-step forecast
def forecast_lstm(model, batch_size, X):
X = X.reshape(1, 1, len(X))
yhat = model.predict(X, batch_size=batch_size)
return yhat[0, 0]
# run a repeated experiment
def experiment(repeats, series, seed):
# transform data to be stationary
raw_values = series.values
diff_values = difference(raw_values, 1)
# transform data to be supervised learning
lag2 = 4
supervised = timeseries_to_supervised(diff_values, lag2)
supervised_values = supervised.values
test_data_num = 32
# split data into train and test-sets
train, test = supervised_values[0:-test_data_num], supervised_values[-test_data_num:]
# transform the scale of the data
print(test)
scaler, train_scaled, test_scaled = scale(train, test)
print(test_scaled)
# run experiment
error_scores = list()
for r in range(repeats):
# fit the model
batch_size = 32
t1 = train.shape[0] % batch_size
t2 = test.shape[0] % batch_size
train_trimmed = train_scaled[t1:, :]
lstm_model = fit_lstm(train_trimmed, batch_size, 30, 4)
# forecast the entire training dataset to build up state for forecasting
print(train_trimmed)
print(train_trimmed[:, 0])
print(train_trimmed[:, :-1])
# if seed:
# train_reshaped = train_trimmed[:, :-1].reshape(len(train_trimmed), 1, lag2)
# lstm_model.predict(train_reshaped, batch_size=batch_size)
# forecast test dataset
test_reshaped = test_scaled[:, 0:-1]
test_reshaped = test_reshaped.reshape(len(test_reshaped), 1, lag2)
output = lstm_model.predict(test_reshaped, batch_size=batch_size)
predictions = list()
for i in range(len(output)):
yhat = output[i, 0]
X = test_scaled[i, 0:-1]
# invert scaling
yhat = invert_scale(scaler, X, yhat)
# invert differencing
yhat = inverse_difference(raw_values, yhat, len(test_scaled) + 1 - i)
# store forecast
predictions.append(yhat)
# report performance
rmse = sqrt(mean_squared_error(raw_values[-test_data_num:], predictions))
print(predictions, raw_values[-test_data_num:])
print('%d) Test RMSE: %.3f' % (r + 1, rmse))
error_scores.append(rmse)
return error_scores
# load dataset
series = read_csv('data.csv', header=0, index_col=0, squeeze=True)
# experiment
repeats = 1
results = DataFrame()
# with seeding
with_seed = experiment(repeats, series, True)
results['with-seed'] = with_seed
# without seeding
without_seed = experiment(repeats, series, False)
results['without-seed'] = without_seed
# summarize results
print(results.describe())
# save boxplot
results.boxplot()