Yugon
/
RL_exhibit
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

refactoring

Browse Source
development
Ugo Finnendahl 5 years ago
parent 12a5cb383e
commit 58aa5df430
5 changed files with 324 additions and 415 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 18
      css/src/style.scss
  2. 2
      css/style.min.css
  3. 17
      index.html
  4. 31
      js/rl.js
  5. 629
      js/view.js

18
css/src/style.scss
Unescape View File

@ -1,3 +1,4 @@
// compileCompressed
*{ *{
margin: 0; margin: 0;
padding: 0; padding: 0;
@ -22,12 +23,23 @@ nav{
left: 10px; left: 10px;
} }
button{
margin: 0.3em;
}
.absolute{ .absolute{
position: absolute; position: absolute;
top:0; top:0;
left:0; left:0;
} }
.stage{
display: flex;
justify-content: center;
align-items: center;
height: 100vh;
}
.plot{ .plot{
position: absolute; position: absolute;
top: 2vh; top: 2vh;
@ -63,12 +75,12 @@ nav{
left: 50%; left: 50%;
transform: translateX(-50%) translateY(-100%); transform: translateX(-50%) translateY(-100%);
z-index: 10; z-index: 10;
width: 50%; max-width: 50%;
height: 40%;
background-color: #BBB; background-color: #BBB;
transition: all 1s; transition: all 1s;
&.active{ &.active{
top:50%; top:20%;
transform: translateX(-50%) translateY(-50%); transform: translateX(-50%) translateY(-50%);
} }
} }

2
css/style.min.css vendored
Unescape View File

@ -1 +1 @@
*{margin:0;padding:0}body{margin:0;padding:0;font-family:sans-serif}#container{height:100vh;position:relative}#canvas{height:100%}nav{position:absolute;top:10px;left:10px}.absolute{position:absolute;top:0;left:0}.plot{position:absolute;top:2vh;right:2vw;width:20vw;height:10vw}.sliders{position:absolute;top:20vh;left:2vw;width:20vw}#formula{position:absolute;top:71.5vh;width:90vw}.score{position:absolute;top:7vh;left:2vw;width:20vw}.lightbox{padding:2em;position:absolute;top:0;left:50%;transform:translateX(-50%) translateY(-100%);z-index:10;width:50%;height:40%;background-color:#BBB;transition:all 1s}.lightbox.active{top:50%;transform:translateX(-50%) translateY(-50%)} *{margin:0;padding:0}body{margin:0;padding:0;font-family:sans-serif}#container{height:100vh;position:relative}#canvas{height:100%}nav{position:absolute;top:10px;left:10px}button{margin:0.3em}.absolute{position:absolute;top:0;left:0}.stage{display:flex;justify-content:center;align-items:center;height:100vh}.plot{position:absolute;top:2vh;right:2vw;width:20vw;height:10vw}.sliders{position:absolute;top:20vh;left:2vw;width:20vw}#formula{position:absolute;top:71.5vh;width:90vw}.score{position:absolute;top:7vh;left:2vw;width:20vw}.lightbox{padding:2em;position:absolute;top:0;left:50%;transform:translateX(-50%) translateY(-100%);z-index:10;max-width:50%;background-color:#BBB;transition:all 1s}.lightbox.active{top:20%;transform:translateX(-50%) translateY(-50%)}

17
index.html
Unescape View File

@ -3,7 +3,7 @@
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<script src="https://unpkg.com/konva@4.0.0/konva.min.js"></script> <script src="https://unpkg.com/konva@4.0.0/konva.min.js"></script>
<script src='https://unpkg.com/vue/dist/vue.js'></script> <script src='https://unpkg.com/vue/dist/vue.min.js'></script>
<script src='https://cdn.jsdelivr.net/npm/vue-konva@2.0.11/umd/vue-konva.min.js'></script> <script src='https://cdn.jsdelivr.net/npm/vue-konva@2.0.11/umd/vue-konva.min.js'></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/gsap/1.20.3/TweenMax.min.js"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/gsap/1.20.3/TweenMax.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.1/Chart.min.js"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.1/Chart.min.js"></script>
@ -23,14 +23,9 @@
<body> <body>
<div id="app"> <div id="app">
<light-box :content="lightText" :options="lightOptions" :active="lightpopup"></light-box> <rl-local class="stage" :machine="machine" :maze="maze" :config="stage_config" v-show="isActive('local')"></rl-local>
<v-stage ref="stage" :config="stage_config" v-show="isActive('global')">
<rl-map :machine="machine" :maze="maze" :config="map_config"></rl-map>
</v-stage>
<v-stage ref="stage" :config="stage_config" v-show="isActive('local')"> <rl-map class="stage" :machine="machine" :maze="maze" :config="stage_config" v-show="isActive('global')"></rl-map>
<rl-local :machine="machine" :maze="maze" :config="map_config"></rl-local>
</v-stage>
<line-chart css-classes="plot" :chart-data="datacollection" :options="plot_options" v-show="isActive('plot')"></line-chart> <line-chart css-classes="plot" :chart-data="datacollection" :options="plot_options" v-show="isActive('plot')"></line-chart>
@ -45,13 +40,11 @@
</div> </div>
<div class="score" v-show="isActive('score')"> <div class="score" v-show="isActive('score')">
<h1>Current Score</h1> <h1>Current Energy</h1>
<h2>{{machine.score}}</h2> <h2>{{machine.score}}</h2>
</div> </div>
<nav class="navi" v-show="isActive('navi')"> <navi-gation :options="navigation" v-show="isActive('navi')"></navi-gation>
<button v-for="(item, key) in navigation" v-on:click="item">{{ key }}</button>
</nav>
</div> </div>

31
js/rl.js
Unescape View File

@ -21,6 +21,10 @@ class RL_machine {
this.epsilon = epsilon; this.epsilon = epsilon;
this.q_table = this.actions_per_state.map((c) => c.reduce((o,n) => {o[n]=0; return o},{})); this.q_table = this.actions_per_state.map((c) => c.reduce((o,n) => {o[n]=0; return o},{}));
this.reset_machine(); this.reset_machine();
this.callback = null;
}
setCallback(cb){
this.callback = cb;
} }
reset_machine(){ reset_machine(){
for (var q in this.q_table){ for (var q in this.q_table){
@ -34,13 +38,20 @@ class RL_machine {
this.state = this.start_state; this.state = this.start_state;
this.score = this.start_score; this.score = this.start_score;
} }
new_episode(){ new_episode(reason = "failed"){
// add_new_episode_callback const reset = () => {
this.episode++; this.episode++;
this.score_history.push(this.score); this.score_history.push(this.score);
this.state = this.start_state; this.state = this.start_state;
this.score = this.start_score; this.score = this.start_score;
} }
// add_new_episode_callback
if (!this.running && this.callback) {
this.callback(reason).then((p) => reset());
} else {
reset();
}
}
auto_step(){ auto_step(){
if (Math.random() < this.epsilon){ if (Math.random() < this.epsilon){
return this.step(choose(Object.keys(this.q_table[this.state]))); return this.step(choose(Object.keys(this.q_table[this.state])));
@ -55,23 +66,11 @@ class RL_machine {
this.state = this.update_q_table(this.state, action); this.state = this.update_q_table(this.state, action);
// add_new_step_callback // add_new_step_callback
if (this.end_states.indexOf(this.state) >= 0) { if (this.end_states.indexOf(this.state) >= 0) {
var succ_event = new CustomEvent("episode",{ this.new_episode("success");
detail: "success"
});
if (!this.running) {
window.dispatchEvent(succ_event);
}
this.new_episode();
return 2 return 2
} }
if (this.score <= this.end_score){ if (this.score <= this.end_score){
var fail_event = new CustomEvent("episode",{ this.new_episode("failed");
detail: "failed"
});
if (!this.running) {
window.dispatchEvent(fail_event);
}
this.new_episode();
return 2 return 2
} }
return 1 return 1

629
js/view.js
Unescape View File

@ -1,3 +1,21 @@
// ----------------------------------------------------------------------------
// ------------------------------- Utils --------------------------------------
// ----------------------------------------------------------------------------
function defer() {
var res, rej;
var promise = new Promise((resolve, reject) => {
res = resolve;
rej = reject;
});
promise.resolve = res;
promise.reject = rej;
return promise;
}
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// -------------------------------- Plot -------------------------------------- // -------------------------------- Plot --------------------------------------
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
@ -44,40 +62,30 @@ Vue.component('line-chart', {
// --------------------------------- Map -------------------------------------- // --------------------------------- Map --------------------------------------
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
function set_images($this){ var MapBase = Vue.component('MapBase', {
props: ['machine', 'maze', 'config'],
data: function () {
return {
robot_image: null,
energy_image: null,
}
},
created() {
var $this = this;
const robot_image = new window.Image(); const robot_image = new window.Image();
robot_image.src = "img/robot.png"; robot_image.src = "img/robot.png";
// robot_image.src = "https://konvajs.org/assets/yoda.jpg";
robot_image.onload = () => { robot_image.onload = () => {
// set image only when it is loaded
$this.robot_image = robot_image; $this.robot_image = robot_image;
}; };
const energy_image = new window.Image(); const energy_image = new window.Image();
energy_image.src = "img/station.png"; energy_image.src = "img/station.png";
// energy_image.src = "https://konvajs.org/assets/yoda.jpg";
energy_image.onload = () => { energy_image.onload = () => {
// set image only when it is loaded
$this.energy_image = energy_image; $this.energy_image = energy_image;
}; };
}
var palette = ['#d2000d', '#d30512', '#d40a17', '#d50f1c', '#d61420', '#d71a25', '#d71f2a', '#d8242f', '#d92934', '#da2e39', '#db333d', '#dc3842', '#dd3d47', '#de424c', '#df4751', '#e04d56', '#e0525a', '#e1575f', '#e25c64', '#e36169', '#e4666e', '#e56b73', '#e67077', '#e7757c', '#e87a81', '#e98086', '#e9858b', '#ea8a90', '#eb8f95', '#ec9499', '#ed999e', '#ee9ea3', '#efa3a8', '#f0a8ad', '#f1adb2', '#f2b3b6', '#f2b8bb', '#f3bdc0', '#f4c2c5', '#f5c7ca', '#f6cccf', '#f7d1d3', '#f8d6d8', '#f9dbdd', '#fae0e2', '#fbe6e7', '#fbebec', '#fcf0f0', '#fdf5f5', '#fefafa', '#ffffff', '#fafcfa', '#f5f9f5', '#f0f6f0', '#ebf3ec', '#e6f1e7', '#e1eee2', '#dcebdd', '#d7e8d8', '#d3e5d3', '#cee2cf', '#c9dfca', '#c4dcc5', '#bfd9c0', '#bad6bb', '#b5d4b6', '#b0d1b2', '#abcead', '#a6cba8', '#a1c8a3', '#9cc59e', '#97c299', '#92bf95', '#8dbc90', '#88b98b', '#84b786', '#7fb481', '#7ab17c', '#75ae77', '#70ab73', '#6ba86e', '#66a569', '#61a264', '#5c9f5f', '#579c5a', '#529a56', '#4d9751', '#48944c', '#439147', '#3e8e42', '#398b3d', '#348839', '#308534', '#2b822f', '#267f2a', '#217d25', '#1c7a20', '#17771c', '#127417', '#0d7112', '#086e0d']
Vue.component('rl-map', {
props: ['machine', 'maze', 'config'],
data: function () {
return {
robot_image: null,
energy_image: null,
}
},
created() {
set_images(this);
}, },
computed: { computed: {
main_config: function(){ main_config: function(){
return { return {
...this.config,
offset: { offset: {
x: -(this.config.width-this.base_size*this.maze.width)/2, x: -(this.config.width-this.base_size*this.maze.width)/2,
y: -(this.config.height-this.base_size*this.maze.height)/2, y: -(this.config.height-this.base_size*this.maze.height)/2,
@ -88,9 +96,13 @@ Vue.component('rl-map', {
return { return {
height: this.base_size, height: this.base_size,
width: this.base_size, width: this.base_size,
x: this.base_size * this.machine.state.x, x: this.center,
y: this.base_size * this.machine.state.y, y: this.center,
image: this.robot_image, image: this.robot_image,
offset:{
x: this.base_size/2,
y: this.base_size/2,
}
} }
}, },
energy_config: function() { energy_config: function() {
@ -104,26 +116,12 @@ Vue.component('rl-map', {
image: this.energy_image, image: this.energy_image,
} }
}, },
base_size: function() {
return Math.min(this.config.height/this.maze.height, this.config.width/this.maze.width);
},
strokeW: function() { strokeW: function() {
return this.base_size / 50; return this.base_size / 50;
}, },
extreme_q_values: function(){ base_size: function() {
var max = -10*30; return Math.min(this.config.height/this.maze.height, this.config.width/this.maze.width);
var min = 10*30; },
for (field in this.q_table) {
for (key in this.q_table[field]){
if (this.q_table[field][key]<min){
min = this.q_table[field][key];
} else if (this.q_table[field][key]>max){
max = this.q_table[field][key];
}
}
}
return {min:min,max:max};
}
}, },
methods: { methods: {
get_tile_type: function(state) { get_tile_type: function(state) {
@ -143,6 +141,86 @@ Vue.component('rl-map', {
y: this.base_size * pos.y+this.base_size/2, y: this.base_size * pos.y+this.base_size/2,
} }
}, },
get_tile_config: function(t_type) {
const layout = {
width: this.base_size,
height: this.base_size,
stroke: '#ddd',
strokeWidth: this.strokeW,
offset: {
x: this.base_size/2,
y: this.base_size/2,
}
};
switch (t_type) {
case tile.regular:
return {
...layout,
fill: '#fff',
opacity: 1,
}
case tile.end:
return {
...layout,
fill: '#0eb500',
opacity: 1,
}
case tile.start:
return {
...layout,
fill: '#ff0008',
opacity: 1,
}
case tile.dangerous:
return {
...layout,
fill: '#FF7B17',
opacity: 1,
}
case tile.wall:
return {
...layout,
fill: '#000000',
opacity: 1,
}
}
},
},
})
//-----------------------------------------------------------------------------
var palette = ['#d2000d', '#d30512', '#d40a17', '#d50f1c', '#d61420', '#d71a25', '#d71f2a', '#d8242f', '#d92934', '#da2e39', '#db333d', '#dc3842', '#dd3d47', '#de424c', '#df4751', '#e04d56', '#e0525a', '#e1575f', '#e25c64', '#e36169', '#e4666e', '#e56b73', '#e67077', '#e7757c', '#e87a81', '#e98086', '#e9858b', '#ea8a90', '#eb8f95', '#ec9499', '#ed999e', '#ee9ea3', '#efa3a8', '#f0a8ad', '#f1adb2', '#f2b3b6', '#f2b8bb', '#f3bdc0', '#f4c2c5', '#f5c7ca', '#f6cccf', '#f7d1d3', '#f8d6d8', '#f9dbdd', '#fae0e2', '#fbe6e7', '#fbebec', '#fcf0f0', '#fdf5f5', '#fefafa', '#ffffff', '#fafcfa', '#f5f9f5', '#f0f6f0', '#ebf3ec', '#e6f1e7', '#e1eee2', '#dcebdd', '#d7e8d8', '#d3e5d3', '#cee2cf', '#c9dfca', '#c4dcc5', '#bfd9c0', '#bad6bb', '#b5d4b6', '#b0d1b2', '#abcead', '#a6cba8', '#a1c8a3', '#9cc59e', '#97c299', '#92bf95', '#8dbc90', '#88b98b', '#84b786', '#7fb481', '#7ab17c', '#75ae77', '#70ab73', '#6ba86e', '#66a569', '#61a264', '#5c9f5f', '#579c5a', '#529a56', '#4d9751', '#48944c', '#439147', '#3e8e42', '#398b3d', '#348839', '#308534', '#2b822f', '#267f2a', '#217d25', '#1c7a20', '#17771c', '#127417', '#0d7112', '#086e0d']
Vue.component('rl-map', {
extends: MapBase,
computed: {
robot_config: function() {
return {
height: this.base_size,
width: this.base_size,
x: this.base_size * this.machine.state.x,
y: this.base_size * this.machine.state.y,
image: this.robot_image,
}
},
extreme_q_values: function(){
var max = -10*30;
var min = 10*30;
for (field in this.q_table) {
for (key in this.q_table[field]){
if (this.q_table[field][key]<min){
min = this.q_table[field][key];
} else if (this.q_table[field][key]>max){
max = this.q_table[field][key];
}
}
}
return {min: min, max: max};
}
},
methods: {
get_q_text_config: function (val, i) { get_q_text_config: function (val, i) {
var off, key; var off, key;
switch (i) { switch (i) {
@ -225,10 +303,6 @@ Vue.component('rl-map', {
context.lineTo($this.base_size/2-stumpf, -width/2); context.lineTo($this.base_size/2-stumpf, -width/2);
context.lineTo($this.base_size/2-stumpf-arrow_l, -width/2); context.lineTo($this.base_size/2-stumpf-arrow_l, -width/2);
context.lineTo($this.base_size/2-stumpf-arrow_l, width/2); context.lineTo($this.base_size/2-stumpf-arrow_l, width/2);
// context.moveTo(0, 0);
// context.lineTo($this.base_size / 2, $this.base_size / 2);
// context.lineTo($this.base_size / 2, -$this.base_size / 2);
// context.lineTo(0, 0);
context.closePath(); context.closePath();
// (!) Konva specific method, it is very important // (!) Konva specific method, it is very important
context.fillStrokeShape(shape); context.fillStrokeShape(shape);
@ -239,90 +313,23 @@ Vue.component('rl-map', {
rotation: rot, rotation: rot,
} }
}, },
get_tile_config: function(i, t_type, local = false) {
// var pos = this.s2p(i);
var over = {};
// not in plus
if (local) {
if (!this.in_plus(this.machine.s2p(i), {
x: Math.round(this.machine.state.x),
y: Math.round(this.machine.state.y)
})) {
over = {
opacity: 0,
fill: "#eee"
};
} else if (i != this.p2s(Math.round(this.machine.state.x), Math.round(this.machine.state.y))) {
over = {
opacity: 1,
fill: "#eee"
};
}
}
const layout = {
width: this.base_size,
height: this.base_size,
stroke: '#ddd',
strokeWidth: this.strokeW,
offset: {
x: this.base_size/2,
y: this.base_size/2,
}
};
switch (t_type) {
case tile.regular:
return {
...layout,
fill: '#fff',
opacity: 1,
...over,
}
case tile.end:
return {
...layout,
fill: '#0eb500',
opacity: 1,
...over,
}
case tile.start:
return {
...layout,
fill: '#ff0008',
opacity: 1,
...over,
}
case tile.dangerous:
return {
...layout,
fill: '#FF7B17',
opacity: 1,
...over,
}
case tile.wall:
return {
...layout,
fill: '#000000',
opacity: 1,
...over,
}
}
},
}, },
template: template:
`<v-layer ref="map_layer"> `<v-stage ref="stage" :config="config">
<v-group ref="map_group" :config="main_config"> <v-layer ref="map_layer" :config="main_config">
<v-group v-for="(t_type, idx) in maze.map.flat()" :config="get_field_config(idx)"> <v-group ref="map_group">
<v-rect :config="get_tile_config(idx, t_type)"></v-rect> <v-group :key="'tile'+idx" v-for="(t_type, idx) in maze.map.flat()" :config="get_field_config(idx)">
<v-rect :config="get_tile_config(t_type)"></v-rect>
<v-image :config="energy_config" v-if="t_type==8"></v-image> <v-image :config="energy_config" v-if="t_type==8"></v-image>
</v-group> </v-group>
<v-group v-for="(action,idx) in machine.q_table" :config="get_field_config(idx)"> <v-group :key="'qgroup'+idx" v-for="(action, idx) in machine.q_table" :config="get_field_config(idx)">
<v-shape v-for="(value, key) in action" :config="get_triangle_config(value, key)"></v-shape> <v-shape :key="'qvalshape'+idx+key" v-for="(value, key) in action" :config="get_triangle_config(value, key)"></v-shape>
<v-text v-for="i in 4" :config="get_q_text_config(action,i)"></v-text> <v-text :key="'qval'+idx+i" v-for="i in 4" :config="get_q_text_config(action,i)"></v-text>
</v-group> </v-group>
<v-image :config="robot_config"></v-image> <v-image :config="robot_config"></v-image>
</v-group> </v-group>
</v-layer>` </v-layer>
</v-stage>`
}) })
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
@ -330,27 +337,17 @@ Vue.component('rl-map', {
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
Vue.component('rl-local', { Vue.component('rl-local', {
props: ['machine', 'maze', 'config'], extends: MapBase,
data: function () {
return {
robot_image: null,
energy_image: null,
}
},
created() {
set_images(this);
},
computed: { computed: {
main_config: function(){ main_config: function(){
return { return {
...this.config,
offset: { offset: {
x: -(this.config.width-this.base_size*3)/2, x: -(this.config.width-this.base_size*3)/2,
y: -(this.config.height-this.base_size*3)/2, y: -(this.config.height-this.base_size*3)/2,
} }
} }
}, },
map_config: function() { local_config: function() {
return { return {
x: -(this.machine.state.x)*this.base_size, x: -(this.machine.state.x)*this.base_size,
y: -(this.machine.state.y)*this.base_size, y: -(this.machine.state.y)*this.base_size,
@ -360,86 +357,55 @@ Vue.component('rl-local', {
} }
} }
}, },
robot_config: function() {
return {
height: this.base_size,
width: this.base_size,
x: this.center,
y: this.center,
image: this.robot_image,
offset:{
x: this.base_size/2,
y: this.base_size/2,
}
}
},
energy_config: function() {
return {
height: this.base_size,
width: this.base_size,
offset: {
x: this.base_size/2,
y: this.base_size/2
},
image: this.energy_image,
}
},
base_size: function() { base_size: function() {
return Math.min(this.config.height/3, this.config.width/3); return Math.min(this.config.height/3, this.config.width/3);
}, },
center: function() { center: function() {
return 3*this.base_size / 2; return 3*this.base_size / 2;
}, },
strokeW: function() { local_area: function() {
return this.base_size / 50; const x = Math.round(this.machine.state.x);
const y = Math.round(this.machine.state.y);
let arr = [[x,y-1],[x+1,y],[x,y+1],[x-1,y],[x,y]];
return arr.filter((p) => p[0] < this.maze.width && p[1] < this.maze.height &&
p[0] >= 0 && p[1] >= 0)
.map((p) => [this.maze.map[p[1]][p[0]], p[1]*this.maze.width+p[0]]);
}, },
}, },
methods: { methods: {
get_tile_type: function(state) { end: function(pos){
var pos = this.machine.s2p(state); return this.maze.get_states(tile.end).indexOf(pos) >= 0;
if (pos.y > maze.height) {
return null;
} else if (pos.x > maze.width) {
return null;
} else {
return maze.map[pos.y][pos.x];
}
}, },
in_plus: function(pos1, pos2) { id_to_dir: function(id){
if (Math.abs(pos1.x - pos2.x) + Math.abs(pos1.y - pos2.y) < 2) { switch (id) {
return true; case 0:
return dir.UP;
case 1:
return dir.RIGHT;
case 2:
return dir.DOWN;
case 3:
return dir.LEFT;
default:
return undefined;
} }
return false;
}, },
get_field_config: function(state) { handleMouseEnter(e) {
var pos = this.machine.s2p(state); const stage = e.target.getStage();
return { stage.container().style.cursor = "pointer";
x: this.base_size * pos.x+this.base_size/2,
y: this.base_size * pos.y+this.base_size/2,
}
}, },
get_tile_config: function(i, t_type, local = true) { handleMouseLeave(e) {
const stage = e.target.getStage();
stage.container().style.cursor = "default";
},
get_local_tile_config: function(i, t_type) {
// var pos = this.s2p(i); // var pos = this.s2p(i);
// in plus
var over = {}; var over = {};
// not in plus if (i != this.machine.p2s(Math.round(this.machine.state.x), Math.round(this.machine.state.y)) &&
if (local) { t_type != tile.wall) {
if (!this.in_plus(this.machine.s2p(i), {
x: Math.round(this.machine.state.x),
y: Math.round(this.machine.state.y)
})) {
over = {
opacity: 0,
fill: "#eee"
};
} else if (i != this.machine.p2s(Math.round(this.machine.state.x), Math.round(this.machine.state.y))) {
over = { over = {
opacity: 1,
fill: "#eee"
};
}
}
const layout = {
width: this.base_size, width: this.base_size,
height: this.base_size, height: this.base_size,
stroke: '#ddd', stroke: '#ddd',
@ -447,94 +413,104 @@ Vue.component('rl-local', {
offset: { offset: {
x: this.base_size/2, x: this.base_size/2,
y: this.base_size/2, y: this.base_size/2,
} },
};
switch (t_type) {
case tile.regular:
return {
...layout,
fill: '#fff',
opacity: 1,
...over,
}
case tile.end:
return {
...layout,
fill: '#0eb500',
opacity: 1,
...over,
}
case tile.start:
return {
...layout,
fill: '#ff0008',
opacity: 1,
...over,
}
case tile.dangerous:
return {
...layout,
fill: '#FF7B17',
opacity: 1,
...over,
}
case tile.wall:
return {
...layout,
opacity: 1, opacity: 1,
...over, fill: "#eee",
fill: '#000000',
} }
} }
return over;
}, },
}, },
template: template:
`<v-layer ref="map_layer" :config="main_config"> `<v-stage ref="stage" :config="config">
<v-group ref="map_group" :config="map_config"> <v-layer ref="map_layer" :config="main_config">
<v-group v-for="(t_type, idx) in maze.map.flat()" :config="get_field_config(idx)"> <v-group ref="map_group" :config="local_config">
<v-rect :config="get_tile_config(idx, t_type)"></v-rect> <v-group :key="pair[1]" v-for="(pair, idx) in local_area" :config="get_field_config(pair[1])">
<v-image :config="energy_config" v-if="t_type==8 && idx==Math.round(machine.state.x)+maze.width*Math.round(machine.state.y)"></v-image> <v-rect :config="get_tile_config(pair[0])"></v-rect>
<v-image :config="energy_config" v-if="end(pair[1])"></v-image>
<v-rect :config="get_local_tile_config(pair[1], pair[0])" @click="id_to_dir(idx) && machine.object.step(id_to_dir(idx))" @mouseenter="handleMouseEnter" @mouseleave="handleMouseLeave"></v-rect>
</v-group> </v-group>
</v-group> </v-group>
<v-image :config="robot_config"></v-image> <v-image :config="robot_config"></v-image>
</v-layer>` </v-layer>
</v-stage>`
}) })
Vue.component('navi-gation', {
props: ["options"],
template: `
<nav class="navi">
<button v-for="(item, key) in options" v-on:click="item">{{ key }}</button>
</nav>`
});
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// ------------------------------ lightbox ------------------------------------ // ------------------------------ lightbox ------------------------------------
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
Vue.component('light-box', { var light_box = {
props: ['content', 'options', 'active'], data: {
// data: function () { content: "",
// return { options: [],
// active: false, active: false,
// } },
// }, methods:{
// methods: { close: function(){
// close: function() { this.active = false;
// this.active = false; },
// }, popup: function(content, options){
// open: function() { this.content = content;
// this.active = true; var answer = defer();
// }, var $this = this;
// }, this.options = options.reduce((old, opt) => {
old[opt] = function(){
$this.active = false;
answer.resolve(opt);
}
return old
}, {});
this.active = true;
return answer;
}
},
template: ` template: `
<div class="lightbox" v-bind:class="{ active: active }">{{ content }} <div class="lightbox" v-bind:class="{ active: active }">{{ content }}
<div class="options"> <div class="options">
<button v-for="(item, key) in options" v-on:click="item">{{ key }}</button> <button :key="key" v-for="(item, key) in options" v-on:click="item">{{ key }}</button>
</div> </div>
</div>` </div>`
}) }
const PopupLibrary = {
install(Vue, options = {}) {
const root = new Vue(light_box)
// Mount root Vue instance on new div element added to body
root.$mount(document.body.appendChild(document.createElement('div')))
Vue.prototype.$lightbox = root;
}
}
window.Vue.use(PopupLibrary)
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// -------------------------------- Main -------------------------------------- // -------------------------------- Main --------------------------------------
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
function makeMachineReactive(th, machine){
function make_machine_reactive(th, machine){
var $this = th; var $this = th;
$this.machine.s2p = function(state) {
return {
x: (state % $this.maze.width),
y: Math.floor(state / $this.maze.width),
}
};
$this.machine.p2s = function(x, y) {
return x + y * $this.maze.width;
};
// Score wrapper // Score wrapper
var s = machine.score; var s = machine.score;
$this.machine.score = s; $this.machine.score = s;
@ -565,7 +541,7 @@ function make_machine_reactive(th, machine){
// State wrapper // State wrapper
var s = machine.state; var s = machine.state;
$this.machine.state = $this.s2p(s); $this.machine.state = $this.machine.s2p(s);
Object.defineProperty(machine, 'state', { Object.defineProperty(machine, 'state', {
get: function() { get: function() {
return this._state return this._state
@ -577,21 +553,7 @@ function make_machine_reactive(th, machine){
}); });
machine.state = s; machine.state = s;
$this.machine.s2p = $this.s2p; $this.machine.object.setCallback($this.onNewEpisode);
$this.machine.p2s = $this.p2s;
window.addEventListener("episode", (ev) => {
if (ev.detail == "failed"){
$this.lightText = "Out of battery. The robot will be resetted.";
} else if (ev.detail == "success"){
$this.lightText = "You reached the goal. The robot will be resetted.";
}
$this.lightOptions = {
"ok": () => {$this.lightpopup = false;},
}
$this.lightpopup = true;
})
} }
app = new Vue({ app = new Vue({
@ -600,7 +562,7 @@ app = new Vue({
VueSlider: window['vue-slider-component'], VueSlider: window['vue-slider-component'],
}, },
data: { data: {
state: 0, state: null,
maze: maze, maze: maze,
machine: { machine: {
object: machine, object: machine,
@ -620,21 +582,15 @@ app = new Vue({
}, },
width: 0, width: 0,
height: 0, height: 0,
state: null,
components: [], components: [],
lightText: "",
lightOptions: "",
lightpopup: false,
navigation: {}, navigation: {},
onEnterState: function(){},
onLeaveState: function(){},
}, },
created() { created() {
// Resize handler // Resize handler
window.addEventListener('resize', this.handleResize) window.addEventListener('resize', this.handleResize)
this.handleResize(); this.handleResize();
make_machine_reactive(this, machine); makeMachineReactive(this, machine);
this.state = "init"; this.state = "init";
}, },
destroyed() { destroyed() {
@ -643,14 +599,8 @@ app = new Vue({
computed: { computed: {
stage_config: function() { stage_config: function() {
return { return {
width: this.width, x: 0,
height: this.height, y: 0,
}
},
map_config: function() {
return {
x: this.width*0.25,
y: this.height*0.1,
width: this.width*0.5, width: this.width*0.5,
height: this.height*0.8, height: this.height*0.8,
} }
@ -707,23 +657,16 @@ app = new Vue({
}, },
}, },
methods: { methods: {
s2p: function(state) { onEnterState: function(){},
return { onLeaveState: function(){},
x: (state % this.maze.width),
y: Math.floor(state / this.maze.width),
}
},
p2s: function(x, y) {
return x + y * this.maze.width;
},
handleState: function(s) { handleState: function(s) {
if (!this.machine.object.running) { if (!this.machine.object.running) {
this.machine.state_tween.to(this.machine.state, 0.2, { this.machine.state_tween.to(this.machine.state, 0.2, {
x: this.s2p(s).x, x: this.machine.s2p(s).x,
y: this.s2p(s).y y: this.machine.s2p(s).y
}); });
} else { } else {
this.machine.state = this.s2p(s); this.machine.state = this.machine.s2p(s);
} }
}, },
handleResize: function() { handleResize: function() {
@ -735,23 +678,29 @@ app = new Vue({
}, },
changeState: function(state){ changeState: function(state){
this.components = []; this.components = [];
this.lightText = "";
this.lightOptions = "";
this.lightpopup = false;
this.navigation = {}; this.navigation = {};
this.onEnterState = function(){}; this.onEnterState = function(){};
this.onLeaveState = function(){}; this.onLeaveState = function(){};
this.state = state; this.state = state;
},
onNewEpisode: function(result){
var text;
if (result == "failed"){
text = "Out of battery. The robot will be resetted.";
} else if (result == "success"){
text = "You reached the goal. The robot will be resetted.";
}
return this.$lightbox.popup(text, ["ok"]);
} }
}, },
watch: { watch: {
'machine.learning_rate': function(new_val) { 'machine.learning_rate': function(new_val) {
machine.lr = new_val; machine.lr = new_val;
render_latex(); renderLatex();
}, },
'machine.discount_factor': function(new_val) { 'machine.discount_factor': function(new_val) {
machine.df = new_val; machine.df = new_val;
render_latex(); renderLatex();
}, },
'machine.epsilon': function(new_val) { 'machine.epsilon': function(new_val) {
machine.epsilon = new_val; machine.epsilon = new_val;
@ -764,11 +713,11 @@ app = new Vue({
} }
}) })
function render_latex() { function renderLatex() {
// (1-lr) * Q[state, action] + lr * (reward + gamma * np.max(Q[new_state, :]) // (1-lr) * Q[state, action] + lr * (reward + gamma * np.max(Q[new_state, :])
katex.render(`Q(s,a)\\leftarrow${(1-machine.lr).toFixed(2)}Q(s,a)+${machine.lr.toFixed(2)}(reward + ${machine.df.toFixed(2)}\\max_{a'}(Q(s_{new}, a'))`, document.getElementById('formula'),{displayMode: true,}); katex.render(`Q(s,a)\\leftarrow${(1-machine.lr).toFixed(2)}Q(s,a)+${machine.lr.toFixed(2)}(reward + ${machine.df.toFixed(2)}\\max_{a'}(Q(s_{new}, a'))`, document.getElementById('formula'),{displayMode: true,});
} }
render_latex(); renderLatex();
@ -778,34 +727,23 @@ render_latex();
var StateMgr = { var StateMgr = {
init: { init: {
lightText: `Reinforcement learning (RL) is an area of machine learning concerned with how software agents ought to take actions in an environment so as to maximize some notion of cumulative reward. Reinforcement learning is one of three basic machine learning paradigms, alongside supervised learning and unsupervised learning. (wikipedia)
This exhibit explains how a robot can learn to navigate through a maze in order to reach its destination, before running out of power. At first the robot knows nothing, and learns from each new action (movement) and state (location reached). Slowly it starts to develop an understanding of the maze that will allow it to reach the charging station before it runs out of power. Eventually, it should learn to avoid any detour and reach the charging station in the optimal number of steps.`,
lightOptions: {
"next": () => app.changeState("local"),
},
onEnterState: function () { onEnterState: function () {
this.lightpopup = true; var lightText = `Reinforcement learning (RL) is an area of machine learning concerned with how software agents ought to take actions in an environment so as to maximize some notion of cumulative reward. Reinforcement learning is one of three basic machine learning paradigms, alongside supervised learning and unsupervised learning. (wikipedia)
This exhibit explains how a robot can learn to navigate through a maze in order to reach its destination, before running out of power. At first the robot knows nothing, and learns from each new action (movement) and state (location reached). Slowly it starts to develop an understanding of the maze that will allow it to reach the charging station before it runs out of power. Eventually, it should learn to avoid any detour and reach the charging station in the optimal number of steps.`
this.$lightbox.popup(lightText, ["next"]).then((r) => this.changeState("local"));
}, },
onLeaveState: function () {
this.lightpopup = false;
}
}, },
local: { local: {
components: ["local", "navi", "score"], components: ["local", "navi", "score"],
navigation: { navigation: {
"reset robot": () => machine.reset_machine(), "reset robot": () => machine.reset_machine(),
"continue": () => app.changeState("global"), "continue": null,
},
lightText: "But there is a problem! The robot cannot see the whole maze, it only knows where it is and in which direction it can move. Can you reach the charging station in those conditions? Use the arrows to move",
lightOptions: {
"next": () => {app.lightpopup = false;},
}, },
onEnterState: function () { onEnterState: function () {
this.lightpopup = true; this.navigation.continue = () => this.changeState("global");
var lightText = "But there is a problem! The robot cannot see the whole maze, it only knows where it is and in which direction it can move. Can you reach the charging station in those conditions? Use the arrows to move";
this.$lightbox.popup(lightText, ["next"]);
}, },
onLeaveState: function () {
this.lightpopup = false;
}
}, },
global: { global: {
components: ["global", "sliders", "plot", "navi", "score"], components: ["global", "sliders", "plot", "navi", "score"],
@ -816,44 +754,11 @@ var StateMgr = {
"greedy step!": () => machine.greedy_step(), "greedy step!": () => machine.greedy_step(),
"reset machine": () => machine.reset_machine(), "reset machine": () => machine.reset_machine(),
}, },
lightText: `As a human, you keep track of where you are and how you got there without thinking, which helps you think about what actions you should take next to reach your destination. And you can also just look around! How can then the robot 'think' of the maze, to know which action is the best at every moment? And how can it learn that? It must somehow keep track of where it is, and remember how good or bad was each action at each place in the maze, try new things, and update it's "mental image" of what was a good decision and what not.
Reinforcement Learning uses the concept of a "Q-function", which keeps track of how "good" it expects it to be to take a specific action 'a' from a specific location 's'. This is written as Q(s, a). It also uses a "policy", which determines the best action to take in a given state, and is written as π(s). The robot must learn those functions while it navigates the maze. With each step, the functions are modified by a little bit, until eventually they give it the best strategy to solve the maze.`,
lightOptions: {
"continue": () => {app.lightpopup = false;},
},
onEnterState: function () { onEnterState: function () {
this.lightpopup = true; var lightText = `As a human, you keep track of where you are and how you got there without thinking, which helps you think about what actions you should take next to reach your destination. And you can also just look around! How can then the robot 'think' of the maze, to know which action is the best at every moment? And how can it learn that? It must somehow keep track of where it is, and remember how good or bad was each action at each place in the maze, try new things, and update it's "mental image" of what was a good decision and what not.
Reinforcement Learning uses the concept of a "Q-function", which keeps track of how "good" it expects it to be to take a specific action 'a' from a specific location 's'. This is written as Q(s, a). It also uses a "policy", which determines the best action to take in a given state, and is written as π(s). The robot must learn those functions while it navigates the maze. With each step, the functions are modified by a little bit, until eventually they give it the best strategy to solve the maze.`;
this.$lightbox.popup(lightText, ["continue"]);
}, },
} }
}; };
// ----------------------------------------------------------------------------
// ------------------------------- Dummy --------------------------------------
// ----------------------------------------------------------------------------
// Vue.component('dummy', {
// props: ['config'],
// data: function () {
// return {
// robot_image: null,
// }
// },
// created() {
//
// },
// mounted() {
//
// },
// computed: {
// fun: function() {
// return
// },
// },
// methods: {
// fun: function(arg) {
// },
// },
// template: ``
// })

Write Preview
Loading…
Cancel
Save