@ -1,24 +1,31 @@
// ----------------------------------------------------------------------------
Vue . component ( 'line-chart' , {
// ------------------------------- Utils --------------------------------------
extends : VueChartJs . Line ,
// ----------------------------------------------------------------------------
mixins : [ VueChartJs . mixins . reactiveProp ] ,
props : [ 'options' ] ,
function defer ( ) { // mixins: [VueChartJs.mixins.reactiveData],
var res , rej ; // props: ['options','labels', 'datasets'],
// watch: {
var promise = new Promise ( ( resolve , reject ) => { // 'labels': function(new_val) {
res = resolve ; // this.chartData = {
rej = reject ; // 'labels': new_val,
} ) ; // 'datasets': this.datasets};
// },
promise . resolve = res ; // 'datasets': {
promise . reject = rej ; // deep:true,
// handler: function(new_val) {
// this.chartData = {
// 'labels': this.labels,
// 'datasets': new_val};
// }
// }
// },
mounted ( ) {
this . renderChart ( this . chartData , this . options ) ;
} ,
return promise ; } )
}
// ----------------------------------------------------------------------------
var palette = [ '#00429d' , '#06449d' , '#0d469d' , '#12489d' , '#164a9d' , '#1a4c9c' , '#1d4e9c' , '#20509c' , '#23529c' , '#26549c' , '#28569b' , '#2b589b' , '#2d5a9b' , '#305c9b' , '#325e9a' , '#34609a' , '#37629a' , '#39649a' , '#3b6699' , '#3d6899' , '#3f6999' , '#416b98' , '#436d98' , '#456f97' , '#477197' , '#497397' , '#4b7596' , '#4d7796' , '#4f7995' , '#517b95' , '#537d94' , '#557f94' , '#578193' , '#598392' , '#5b8592' , '#5d8791' , '#5f8991' , '#618a90' , '#638c8f' , '#658e8f' , '#67908e' , '#69928d' , '#6b948d' , '#6d968c' , '#6f988b' , '#719a8b' , '#739c8a' , '#759e89' , '#77a088' , '#79a287' , '#7ba386' , '#7da586' , '#80a785' , '#82a984' , '#84ab83' , '#86ad82' , '#88af81' , '#8ab180' , '#8cb37f' , '#8fb57e' , '#91b77d' , '#93b87c' , '#95ba7a' , '#97bc79' , '#9abe78' , '#9cc077' , '#9ec276' , '#a1c474' , '#a3c673' , '#a5c872' , '#a8c970' , '#aacb6f' , '#accd6e' , '#afcf6c' , '#b1d16b' , '#b4d369' , '#b6d568' , '#b8d766' , '#bbd864' , '#bdda63' , '#c0dc61' , '#c2de5f' , '#c5e05d' , '#c8e25b' , '#cae459' , '#cde657' , '#cfe755' , '#d2e953' , '#d5eb50' , '#d7ed4e' , '#daef4b' , '#ddf049' , '#dff246' , '#e2f443' , '#e5f640' , '#e8f83c' , '#ebfa39' , '#edfb35' , '#f0fd31' , '#f3ff2c' ]
// -------------------------------- Plot --------------------------------------
// ----------------------------------------------------------------------------
Array . prototype . simpleSMA = function ( N ) { Array . prototype . simpleSMA = function ( N ) {
return this . map ( return this . map (
@ -48,163 +55,184 @@ Array.prototype.max = function() {
} ) ; } ) ;
} ; } ;
Vue . component ( 'line-chart' , { app = new Vue ( {
extends : VueChartJs . Line , el : '#app' ,
mixins : [ VueChartJs . mixins . reactiveProp ] , components : {
props : [ 'options' ] , VueSlider : window [ 'vue-slider-component' ]
mounted ( ) {
this . renderChart ( this . chartData , this . options ) ;
} , } ,
data : {
} ) width : 0 ,
height : 0 ,
// ----------------------------------------------------------------------------
q _table : machine . q _table ,
// --------------------------------- Map --------------------------------------
maze : maze ,
// ----------------------------------------------------------------------------
state : {
x : 0 ,
var MapBase = Vue . component ( 'MapBase' , { y : 0
props : [ 'machine' , 'maze' , 'config' ] , } ,
data : function ( ) { state _tween : new TimelineLite ( ) ,
return { score : machine . score ,
robot _image : null , score _history : machine . score _history ,
energy _image : null , labels : [ ] ,
learning _rate : machine . lr ,
discount _factor : machine . df ,
epsilon : machine . epsilon ,
slider _config : {
min : 0 ,
max : 1 ,
duration : 0 ,
interval : 0.01 ,
tooltip : 'none'
} }
} , } ,
created ( ) { created ( ) {
// Resize handler
window . addEventListener ( 'resize' , this . handleResize )
this . handleResize ( ) ;
// State wrapper
var s = machine . state ;
var $this = this ; var $this = this ;
const robot _image = new window . Image ( ) ; this . state = this . s2p ( s ) ;
robot _image . src = "img/robot.png" ; Object . defineProperty ( machine , 'state' , {
robot _image . onload = ( ) => { get : function ( ) {
$this . robot _image = robot _image ; return this . _state
} ;
const energy _image = new window . Image ( ) ;
energy _image . src = "img/station.png" ;
energy _image . onload = ( ) => {
$this . energy _image = energy _image ;
} ;
} , } ,
computed : { set : function ( ne ) {
main _config : function ( ) { this . _state = ne ;
return { $this . handleState ( this . _state ) ;
offset : {
x : - ( this . config . width - this . base _size * this . maze . width ) / 2 ,
y : - ( this . config . height - this . base _size * this . maze . height ) / 2 ,
}
} }
} ) ;
machine . state = s ;
// Score wrapper
var s = machine . score ;
var $this = this ;
this . score = s ;
Object . defineProperty ( machine , 'score' , {
get : function ( ) {
return this . _score
} , } ,
robot _config : function ( ) { set : function ( ne ) {
return { this . _score = ne ;
height : this . base _size , $this . score = ne
width : this . base _size ,
x : this . center ,
y : this . center ,
image : this . robot _image ,
offset : {
x : this . base _size / 2 ,
y : this . base _size / 2 ,
}
} }
} ) ;
machine . score = s ;
// Score history wrapper
var s = machine . score _history ;
var $this = this ;
this . score _history = s ;
Object . defineProperty ( machine , 'score_history' , {
get : function ( ) {
return this . _score _history
} , } ,
energy _config : function ( ) { set : function ( ne ) {
return { this . _score _history = ne ;
height : this . base _size , $this . score _history = ne
width : this . base _size ,
offset : {
x : this . base _size / 2 ,
y : this . base _size / 2
} ,
image : this . energy _image ,
} }
} ) ;
machine . score _history = s ;
} , } ,
strokeW : function ( ) { destroyed ( ) {
return this . base _size / 50 ; window . removeEventListener ( 'resize' , this . handleResize )
} ,
base _size : function ( ) {
return Math . min ( this . config . height / this . maze . height , this . config . width / this . maze . width ) ;
} , } ,
computed : {
datacollection : function ( ) {
return {
labels : Array . from ( Array ( this . score _history . length ) . keys ( ) ) ,
datasets : [ {
label : 'Data One' ,
backgroundColor : 'rgb(0,0,0,0)' ,
data : this . score _history , //.simpleSMA(Math.round(50)),
fill : false ,
borderColor : 'rgb(255, 159, 64)' ,
pointRadius : 1 ,
} , } ,
methods : { // {
get _tile _type : function ( state ) { // label: 'Data One',
var pos = this . machine . s2p ( state ) ; // backgroundColor: 'rgb(0,0,0,0)',
if ( pos . y > maze . height ) { // data: this.score_history.max(),
return null ; // fill: false,
} else if ( pos . x > maze . width ) { // borderColor: 'rgb(64, 159, 255)',
return null ; // pointRadius: 1,
} else { // },
return maze . map [ pos . y ] [ pos . x ] ; ]
} }
} , } ,
get _field _config : function ( state ) { plot _options : function ( ) {
var po s= this . machine . s2p ( state ) ; var $thi s= this ;
return { return {
x : this . base _size * pos . x + this . base _size / 2 , responsive : true ,
y : this . base _size * pos . y + this . base _size / 2 , maintainAspectRatio : false ,
scales : {
xAxes : [ {
// type: 'linear',
ticks : {
maxTicksLimit : 8 ,
maxRotation : 0 ,
} }
} ]
} , } ,
get _tile _config : function ( t _type ) { legend : {
const layout = { display : false
width : this . base _size ,
height : this . base _size ,
stroke : '#ddd' ,
strokeWidth : this . strokeW ,
offset : {
x : this . base _size / 2 ,
y : this . base _size / 2 ,
} }
} ;
switch ( t _type ) {
case tile . regular :
return {
... layout ,
fill : '#fff' ,
opacity : 1 ,
} }
case tile . end : } ,
stage _config : function ( ) {
return { return {
... layout , width : this . width ,
fill : '#0eb500' , height : this . height ,
opacity : 1 ,
} }
case tile . start : } ,
mini _map _config : function ( ) {
return { return {
... layout , x : this . width / 2 - ( this . base _size * ( this . maze . width ) / 2 ) ,
fill : '#ff0008' , y : this . height / 2 - ( this . base _size * ( this . maze . height ) / 2 ) ,
opacity : 1 , scale : {
x : 1 ,
y : 1
} }
case tile . dangerous :
return {
... layout ,
fill : '#FF7B17' ,
opacity : 1 ,
} }
case tile . wall : } ,
local _layer : function ( ) {
return { return {
... layout , x : this . width / 2 ,
fill : '#000000' , y : this . height / 2 ,
opacity : 1 , scale : {
x : 2 ,
y : 2
} }
} }
} , } ,
map _config : function ( ) {
return {
x : this . base _size * ( this . maze . width - this . state . x ) ,
y : this . base _size * ( this . maze . height - this . state . y ) ,
offset : {
x : this . base _size * this . maze . width + this . base _size / 2 ,
y : this . base _size * this . maze . height + this . base _size / 2 ,
}
}
} , } ,
} ) agent _config : function ( ) {
//-----------------------------------------------------------------------------
var palette = [ '#d2000d' , '#d30512' , '#d40a17' , '#d50f1c' , '#d61420' , '#d71a25' , '#d71f2a' , '#d8242f' , '#d92934' , '#da2e39' , '#db333d' , '#dc3842' , '#dd3d47' , '#de424c' , '#df4751' , '#e04d56' , '#e0525a' , '#e1575f' , '#e25c64' , '#e36169' , '#e4666e' , '#e56b73' , '#e67077' , '#e7757c' , '#e87a81' , '#e98086' , '#e9858b' , '#ea8a90' , '#eb8f95' , '#ec9499' , '#ed999e' , '#ee9ea3' , '#efa3a8' , '#f0a8ad' , '#f1adb2' , '#f2b3b6' , '#f2b8bb' , '#f3bdc0' , '#f4c2c5' , '#f5c7ca' , '#f6cccf' , '#f7d1d3' , '#f8d6d8' , '#f9dbdd' , '#fae0e2' , '#fbe6e7' , '#fbebec' , '#fcf0f0' , '#fdf5f5' , '#fefafa' , '#ffffff' , '#fafcfa' , '#f5f9f5' , '#f0f6f0' , '#ebf3ec' , '#e6f1e7' , '#e1eee2' , '#dcebdd' , '#d7e8d8' , '#d3e5d3' , '#cee2cf' , '#c9dfca' , '#c4dcc5' , '#bfd9c0' , '#bad6bb' , '#b5d4b6' , '#b0d1b2' , '#abcead' , '#a6cba8' , '#a1c8a3' , '#9cc59e' , '#97c299' , '#92bf95' , '#8dbc90' , '#88b98b' , '#84b786' , '#7fb481' , '#7ab17c' , '#75ae77' , '#70ab73' , '#6ba86e' , '#66a569' , '#61a264' , '#5c9f5f' , '#579c5a' , '#529a56' , '#4d9751' , '#48944c' , '#439147' , '#3e8e42' , '#398b3d' , '#348839' , '#308534' , '#2b822f' , '#267f2a' , '#217d25' , '#1c7a20' , '#17771c' , '#127417' , '#0d7112' , '#086e0d' ]
Vue . component ( 'rl-map' , {
extends : MapBase ,
computed : {
robot _config : function ( ) {
return { return {
height : this . base _size , sides : 5 ,
width : this . base _size , radius : this . base _size / 3 ,
x : this . base _size * this . machine . state . x , fill : '#00D2FF' ,
y : this . base _size * this . machine . state . y , stroke : 'black' ,
image : this . robot _image , strokeWidth : this . strokeW ,
offset : {
x : - this . base _size / 2 ,
y : - this . base _size / 2
} ,
x : this . base _size * this . state . x ,
y : this . base _size * this . state . y ,
} }
} , } ,
base _size : function ( ) {
return Math . min ( this . stage _config . height * 0.8 / this . maze . height , this . stage _config . width * 0.5 / this . maze . width ) ;
} ,
strokeW : function ( ) {
return this . base _size / 50 ;
} ,
extreme _q _values : function ( ) { extreme _q _values : function ( ) {
var max = - 10 * 30 ; var max = - 10 * 30 ;
var min = 10 * 30 ; var min = 10 * 30 ;
@ -221,6 +249,66 @@ Vue.component('rl-map', {
} }
} , } ,
methods : { methods : {
s2p : function ( state ) {
return {
x : ( state % this . maze . width ) ,
y : Math . floor ( state / this . maze . width ) ,
}
} ,
p2s : function ( x , y ) {
return x + y * this . maze . width ;
} ,
handleResize : function ( ) {
this . width = window . innerWidth ;
this . height = window . innerHeight ;
} ,
handleState : function ( s ) {
if ( ! machine . running ) {
this . state _tween . to ( this . state , 0.2 , {
x : this . s2p ( s ) . x ,
y : this . s2p ( s ) . y
} ) ;
} else {
this . state = this . s2p ( s ) ;
}
// this.hidden_state = s;
} ,
get _grid _line _config : function ( idx , y = false ) {
var offset = this . strokeW / 2 ;
if ( y ) {
var points = [ - offset , Math . round ( idx * this . base _size ) , this . base _size * this . maze . width + offset , Math . round ( idx * this . base _size ) ] ;
} else {
var points = [ Math . round ( idx * this . base _size ) , - offset , Math . round ( idx * this . base _size ) , this . base _size * this . maze . height + offset ] ;
}
return {
points : points ,
stroke : '#ddd' ,
strokeWidth : this . strokeW ,
}
} ,
get _tile _type : function ( state ) {
var pos = this . s2p ( state ) ;
if ( pos . y > maze . height ) {
return null ;
} else if ( pos . x > maze . width ) {
return null ;
} else {
return maze . map [ pos . y ] [ pos . x ] ;
}
} ,
in _plus : function ( pos1 , pos2 ) {
if ( Math . abs ( pos1 . x - pos2 . x ) + Math . abs ( pos1 . y - pos2 . y ) < 2 ) {
return true ;
}
return false ;
} ,
get _field _config : function ( state ) {
var pos = this . s2p ( state ) ;
return {
x : this . base _size * pos . x + this . base _size / 2 ,
y : this . base _size * pos . y + this . base _size / 2 ,
}
} ,
get _q _text _config : function ( val , i ) { get _q _text _config : function ( val , i ) {
var off , key ; var off , key ;
switch ( i ) { switch ( i ) {
@ -260,13 +348,13 @@ Vue.component('rl-map', {
fontSize : this . base _size / 7 , fontSize : this . base _size / 7 ,
fontFamily : 'Calibri' , fontFamily : 'Calibri' ,
fill : 'black' , fill : 'black' ,
text : + val [ key ] . toPrecision ( 3 ) + '' , text : + val [ key ] . toFixed ( 2 ) + '' ,
width : this . base _size - 20 , width : this . base _size - 5 ,
height : this . base _size - 34 , height : this . base _size - 5 ,
... off , ... off ,
offset : { offset : {
x : ( this . base _size - 20 ) / 2 , x : ( this . base _size - 5 ) / 2 ,
y : ( this . base _size - 34 ) / 2 , y : ( this . base _size - 5 ) / 2 ,
} }
} }
} , } ,
@ -287,478 +375,109 @@ Vue.component('rl-map', {
break ; break ;
} }
var $this = this ; var $this = this ;
var norma _value = value > 0 ? ( value + 1000 ) / ( 2000 ) : ( value + 30 ) / 60 ; var norma _value = ( value - this . extreme _q _values . min ) / ( ( this . extreme _q _values . max - this . extreme _q _values . min ) || 1 ) ;
return { return {
sceneFunc : function ( context , shape ) { sceneFunc : function ( context , shape ) {
context . beginPath ( ) ; context . beginPath ( ) ;
var width = $this . base _size / 5 ; context . moveTo ( 0 , 0 ) ;
var arrow _w = $this . base _size / 2 ; context . lineTo ( $this . base _size / 2 , $this . base _size / 2 ) ;
var stumpf = $this . base _size / 6 ; context . lineTo ( $this . base _size / 2 , - $this . base _size / 2 ) ;
var arrow _l = $this . base _size / 5 ; context . lineTo ( 0 , 0 ) ;
context . moveTo ( $this . base _size / 2 - stumpf - arrow _l , width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , arrow _w / 2 ) ;
context . lineTo ( $this . base _size / 2 - 2 , 0 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , - arrow _w / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , - width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf - arrow _l , - width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf - arrow _l , width / 2 ) ;
context . closePath ( ) ; context . closePath ( ) ;
// (!) Konva specific method, it is very important
// (!) Konva specific method, it is very important
context . fillStrokeShape ( shape ) ; context . fillStrokeShape ( shape ) ;
} , } ,
fill : palette [ Math . round ( norma _value * 100 ) ] , fill : palette [ Math . round ( norma _value * 99 ) ] ,
stroke : 'black' , stroke : 'black' ,
strokeWidth : 1 , strokeWidth : 0 ,
rotation : rot , rotation : rot ,
} }
} , } ,
} , get _tile _config : function ( i , t _type , local = false ) {
template : var pos = this . s2p ( i ) ;
` <v-stage ref="stage" :config="config">
< v - layer ref = "map_layer" : config = "main_config" >
< v - group ref = "map_group" >
< v - group : key = "'tile'+idx" v - for = "(t_type, idx) in maze.map.flat()" : config = "get_field_config(idx)" >
< v - rect : config = "get_tile_config(t_type)" > < / v - r e c t >
< v - image : config = "energy_config" v - if = "t_type==8" > < / v - i m a g e >
< / v - g r o u p >
< v - group : key = "'qgroup'+idx" v - for = "(action, idx) in machine.q_table" : config = "get_field_config(idx)" >
< v - shape : key = "'qvalshape'+idx+key" v - for = "(value, key) in action" : config = "get_triangle_config(value, key)" > < / v - s h a p e >
< v - text : key = "'qval'+idx+i" v - for = "i in 4" : config = "get_q_text_config(action,i)" > < / v - t e x t >
< / v - g r o u p >
< v - image : config = "robot_config" > < / v - i m a g e >
< / v - g r o u p >
< / v - l a y e r >
< / v - s t a g e > `
} )
// ----------------------------------------------------------------------------
// -------------------------------- Local -------------------------------------
// ----------------------------------------------------------------------------
Vue . component ( 'rl-local' , {
extends : MapBase ,
computed : {
main _config : function ( ) {
return {
offset : {
x : - ( this . config . width - this . base _size * 3 ) / 2 ,
y : - ( this . config . height - this . base _size * 3 ) / 2 ,
}
}
} ,
local _config : function ( ) {
return {
x : - ( this . machine . state . x ) * this . base _size ,
y : - ( this . machine . state . y ) * this . base _size ,
offset : {
x : - this . base _size ,
y : - this . base _size ,
}
}
} ,
base _size : function ( ) {
return Math . min ( this . config . height / 3 , this . config . width / 3 ) ;
} ,
center : function ( ) {
return 3 * this . base _size / 2 ;
} ,
local _area : function ( ) {
const x = Math . round ( this . machine . state . x ) ;
const y = Math . round ( this . machine . state . y ) ;
let arr = [ [ x , y - 1 ] , [ x + 1 , y ] , [ x , y + 1 ] , [ x - 1 , y ] , [ x , y ] ] ;
return arr . filter ( ( p ) => p [ 0 ] < this . maze . width && p [ 1 ] < this . maze . height &&
p [ 0 ] >= 0 && p [ 1 ] >= 0 )
. map ( ( p ) => [ this . maze . map [ p [ 1 ] ] [ p [ 0 ] ] , p [ 1 ] * this . maze . width + p [ 0 ] ] ) ;
} ,
} ,
methods : {
end : function ( pos ) {
return this . maze . get _states ( tile . end ) . indexOf ( pos ) >= 0 ;
} ,
id _to _dir : function ( id ) {
switch ( id ) {
case 0 :
return dir . UP ;
case 1 :
return dir . RIGHT ;
case 2 :
return dir . DOWN ;
case 3 :
return dir . LEFT ;
default :
return undefined ;
}
} ,
handleMouseEnter ( e ) {
const stage = e . target . getStage ( ) ;
stage . container ( ) . style . cursor = "pointer" ;
} ,
handleMouseLeave ( e ) {
const stage = e . target . getStage ( ) ;
stage . container ( ) . style . cursor = "default" ;
} ,
get _local _tile _config : function ( i , t _type ) {
// var pos = this.s2p(i);
// in plus
var over = { } ; var over = { } ;
if ( i != this . machine . p2s ( Math . round ( this . machine . state . x ) , Math . round ( this . machine . state . y ) ) && // not in plus
t _type != tile . wall ) { if ( local ) {
if ( ! this . in _plus ( this . s2p ( i ) , {
x : Math . round ( this . state . x ) ,
y : Math . round ( this . state . y )
} ) ) {
over = { over = {
opacity : 0 ,
fill : "#eee"
} ;
} else if ( i != this . p2s ( Math . round ( this . state . x ) , Math . round ( this . state . y ) ) ) {
over = {
opacity : 1 ,
fill : "#eee"
} ;
}
}
const layout = {
x : this . base _size * pos . x ,
y : this . base _size * pos . y ,
width : this . base _size , width : this . base _size ,
height : this . base _size , height : this . base _size ,
stroke : '#ddd' , stroke : '#ddd' ,
strokeWidth : this . strokeW , strokeWidth : this . strokeW ,
offset : {
x : this . base _size / 2 ,
y : this . base _size / 2 ,
} ,
opacity : 1 ,
fill : "#eee" ,
}
}
return over ;
} ,
} ,
template :
` <v-stage ref="stage" :config="config">
< v - layer ref = "map_layer" : config = "main_config" >
< v - group ref = "map_group" : config = "local_config" >
< v - group : key = "pair[1]" v - for = "(pair, idx) in local_area" : config = "get_field_config(pair[1])" >
< v - rect : config = "get_tile_config(pair[0])" > < / v - r e c t >
< v - image : config = "energy_config" v - if = "end(pair[1])" > < / v - i m a g e >
< v - rect : config = "get_local_tile_config(pair[1], pair[0])" @ click = "id_to_dir(idx) && machine.object.step(id_to_dir(idx))" @ mouseenter = "handleMouseEnter" @ mouseleave = "handleMouseLeave" > < / v - r e c t >
< / v - g r o u p >
< / v - g r o u p >
< v - image : config = "robot_config" > < / v - i m a g e >
< / v - l a y e r >
< / v - s t a g e > `
} )
Vue . component ( 'navi-gation' , {
props : [ "options" ] ,
template : `
< nav class = "navi" >
< button v - for = "(item, key) in options" v - on : click = "item" > { { key } } < / b u t t o n >
< / n a v > `
} ) ;
// ----------------------------------------------------------------------------
// ------------------------------ lightbox ------------------------------------
// ----------------------------------------------------------------------------
var light _box = {
data : {
content : "" ,
options : [ ] ,
active : false ,
} ,
methods : {
close : function ( ) {
this . active = false ;
} ,
popup : function ( content , options ) {
this . content = content ;
var answer = defer ( ) ;
var $this = this ;
this . options = options . reduce ( ( old , opt ) => {
old [ opt ] = function ( ) {
$this . active = false ;
answer . resolve ( opt ) ;
}
return old
} , { } ) ;
this . active = true ;
return answer ;
}
} ,
template : `
< div class = "lightbox" v - bind : class = "{ active: active }" > { { content } }
< div class = "options" >
< button : key = "key" v - for = "(item, key) in options" v - on : click = "item" > { { key } } < / b u t t o n >
< / d i v >
< / d i v > `
}
const PopupLibrary = {
install ( Vue , options = { } ) {
const root = new Vue ( light _box )
// Mount root Vue instance on new div element added to body
root . $mount ( document . body . appendChild ( document . createElement ( 'div' ) ) )
Vue . prototype . $lightbox = root ;
}
}
window . Vue . use ( PopupLibrary )
// ----------------------------------------------------------------------------
// -------------------------------- Main --------------------------------------
// ----------------------------------------------------------------------------
function makeMachineReactive ( th , machine ) {
var $this = th ;
$this . machine . s2p = function ( state ) {
return {
x : ( state % $this . maze . width ) ,
y : Math . floor ( state / $this . maze . width ) ,
}
} ;
$this . machine . p2s = function ( x , y ) {
return x + y * $this . maze . width ;
} ; } ;
switch ( t _type ) {
// Score wrapper
case tile . regular :
var s = machine . score ;
$this . machine . score = s ;
Object . defineProperty ( machine , 'score' , {
get : function ( ) {
return this . _score
} ,
set : function ( ne ) {
this . _score = ne ;
$this . machine . score = ne
}
} ) ;
machine . score = s ;
// Score history wrapper
var s = machine . score _history ;
$this . machine . score _history = s ;
Object . defineProperty ( machine , 'score_history' , {
get : function ( ) {
return this . _score _history
} ,
set : function ( ne ) {
this . _score _history = ne ;
$this . machine . score _history = ne
}
} ) ;
machine . score _history = s ;
// State wrapper
var s = machine . state ;
$this . machine . state = $this . machine . s2p ( s ) ;
Object . defineProperty ( machine , 'state' , {
get : function ( ) {
return this . _state
} ,
set : function ( ne ) {
this . _state = ne ;
$this . handleState ( this . _state ) ;
}
} ) ;
machine . state = s ;
$this . machine . object . setCallback ( $this . onNewEpisode ) ;
}
app = new Vue ( {
el : '#app' ,
components : {
VueSlider : window [ 'vue-slider-component' ] ,
} ,
data : {
state : null ,
maze : maze ,
machine : {
object : machine ,
q _table : machine . q _table ,
state : {
x : 0 ,
y : 0 ,
} ,
state _tween : new TimelineLite ( ) ,
learning _rate : machine . lr ,
discount _factor : machine . df ,
epsilon : machine . epsilon ,
score : machine . score ,
score _history : machine . score _history ,
s2p : null ,
p2s : null ,
} ,
width : 0 ,
height : 0 ,
components : [ ] ,
navigation : { } ,
} ,
created ( ) {
// Resize handler
window . addEventListener ( 'resize' , this . handleResize )
this . handleResize ( ) ;
makeMachineReactive ( this , machine ) ;
this . state = "init" ;
} ,
destroyed ( ) {
window . removeEventListener ( 'resize' , this . handleResize )
} ,
computed : {
stage _config : function ( ) {
return { return {
x : 0 , ... layout ,
y : 0 , fill : '#fff' ,
width : this . width * 0.5 , opacity : 1 ,
height : this . height * 0.8 , ... over ,
} }
} , case tile . end :
slider _config : function ( ) {
return { return {
min : 0 , ... layout ,
max : 1 , fill : '#0eb500' ,
duration : 0 , opacity : 1 ,
interval : 0.01 , ... over ,
tooltip : 'none'
} }
} , case tile . start :
datacollection : function ( ) {
return { return {
labels : Array . from ( Array ( this . machine . score _history . length ) . keys ( ) ) , ... layout ,
datasets : [ { fill : '#ff0008' ,
label : 'Data One' , opacity : 1 ,
backgroundColor : 'rgb(0,0,0,0)' , ... over ,
data : this . machine . score _history , //.simpleSMA(Math.round(50)),
fill : false ,
borderColor : 'rgb(255, 159, 64)' ,
pointRadius : 1 ,
} ,
// {
// label: 'Data One',
// backgroundColor: 'rgb(0,0,0,0)',
// data: this.score_history.max(),
// fill: false,
// borderColor: 'rgb(64, 159, 255)',
// pointRadius: 1,
// },
]
} }
} , case tile . dangerous :
plot _options : function ( ) {
var $this = this ;
return { return {
responsive : true , ... layout ,
maintainAspectRatio : false , fill : '#FF7B17' ,
scales : { opacity : 1 ,
xAxes : [ { ... over ,
// type: 'linear',
ticks : {
maxTicksLimit : 8 ,
maxRotation : 0 ,
}
} ]
} ,
legend : {
display : false
}
} }
} , case tile . wall :
} , return {
methods : { ... layout ,
onEnterState : function ( ) { } , fill : '#000000' ,
onLeaveState : function ( ) { } , opacity : 1 ,
handleState : function ( s ) { ... over ,
if ( ! this . machine . object . running ) {
this . machine . state _tween . to ( this . machine . state , 0.2 , {
x : this . machine . s2p ( s ) . x ,
y : this . machine . s2p ( s ) . y
} ) ;
} else {
this . machine . state = this . machine . s2p ( s ) ;
} }
} ,
handleResize : function ( ) {
this . width = window . innerWidth ;
this . height = window . innerHeight ;
} ,
isActive : function ( what ) {
return this . components . indexOf ( what ) >= 0 ;
} ,
changeState : function ( state ) {
this . components = [ ] ;
this . navigation = { } ;
this . onEnterState = function ( ) { } ;
this . onLeaveState = function ( ) { } ;
this . state = state ;
} ,
onNewEpisode : function ( result ) {
var text ;
if ( result == "failed" ) {
text = "Out of battery. The robot will be resetted." ;
} else if ( result == "success" ) {
text = "You reached the goal. The robot will be resetted." ;
} }
return this . $lightbox . popup ( text , [ "ok" ] ) ;
} }
} , } ,
watch : { watch : {
'machine.learning_rate' : function ( new _val ) { learning _rate : function ( new _val ) {
machine . lr = new _val ; machine . lr = new _val ;
renderL atex ( ) ; render _latex ( ) ;
} , } ,
'machine.discount_factor' : function ( new _val ) { discount _factor : function ( new _val ) {
machine . df = new _val ; machine . df = new _val ;
renderL atex ( ) ; render _latex ( ) ;
} , } ,
'machine.epsilon' : function ( new _val ) { epsilon : function ( new _val ) {
machine . epsilon = new _val ; machine . epsilon = new _val ;
} , }
state : function ( state ) {
this . onLeaveState ( ) ;
Object . assign ( this , StateMgr [ state ] ) ;
this . onEnterState ( ) ;
} ,
} }
} ) } )
function renderL atex ( ) { function render _latex ( ) {
// (1-lr) * Q[state, action] + lr * (reward + gamma * np.max(Q[new_state, :])
// (1-lr) * Q[state, action] + lr * (reward + gamma * np.max(Q[new_state, :])
katex . render ( ` Q(s,a) \\ leftarrow ${ ( 1 - machine . lr ) . toFixed ( 2 ) } Q(s,a)+ ${ machine . lr . toFixed ( 2 ) } (reward + ${ machine . df . toFixed ( 2 ) } \\ max_{a'}(Q(s_{new}, a')) ` , document . getElementById ( 'formula' ) , { displayMode : true , } ) ; katex . render ( ` Q(s,a) \\ leftarrow ${ ( 1 - machine . lr ) . toFixed ( 2 ) } Q(s,a)+ ${ machine . lr . toFixed ( 2 ) } (reward + ${ machine . df . toFixed ( 2 ) } * \\ max_a(Q(s', a)) ` , document . getElementById ( 'test' ) , { displayMode : true , } ) ;
}
renderLatex ( ) ;
// ----------------------------------------------------------------------------
// ------------------------------ StateMgr ------------------------------------
// ----------------------------------------------------------------------------
var StateMgr = {
init : {
onEnterState : function ( ) {
var lightText = ` Reinforcement learning (RL) is an area of machine learning concerned with how software agents ought to take actions in an environment so as to maximize some notion of cumulative reward. Reinforcement learning is one of three basic machine learning paradigms, alongside supervised learning and unsupervised learning. (wikipedia)
This exhibit explains how a robot can learn to navigate through a maze in order to reach its destination , before running out of power . At first the robot knows nothing , and learns from each new action ( movement ) and state ( location reached ) . Slowly it starts to develop an understanding of the maze that will allow it to reach the charging station before it runs out of power . Eventually , it should learn to avoid any detour and reach the charging station in the optimal number of steps . `
this . $lightbox . popup ( lightText , [ "next" ] ) . then ( ( r ) => this . changeState ( "local" ) ) ;
} ,
} ,
local : {
components : [ "local" , "navi" , "score" ] ,
navigation : {
"reset robot" : ( ) => machine . reset _machine ( ) ,
"continue" : null ,
} ,
onEnterState : function ( ) {
this . navigation . continue = ( ) => this . changeState ( "global" ) ;
var lightText = "But there is a problem! The robot cannot see the whole maze, it only knows where it is and in which direction it can move. Can you reach the charging station in those conditions? Use the arrows to move" ;
this . $lightbox . popup ( lightText , [ "next" ] ) ;
} ,
} ,
global : {
components : [ "global" , "sliders" , "plot" , "navi" , "score" ] ,
navigation : {
"run 1 episode!" : ( ) => machine . run ( 1 ) ,
"run 100 episodes!" : ( ) => machine . run ( 100 ) ,
"auto step!" : ( ) => machine . auto _step ( ) ,
"greedy step!" : ( ) => machine . greedy _step ( ) ,
"reset machine" : ( ) => machine . reset _machine ( ) ,
} ,
onEnterState : function ( ) {
var lightText = ` As a human, you keep track of where you are and how you got there without thinking, which helps you think about what actions you should take next to reach your destination. And you can also just look around! How can then the robot 'think' of the maze, to know which action is the best at every moment? And how can it learn that? It must somehow keep track of where it is, and remember how good or bad was each action at each place in the maze, try new things, and update it's "mental image" of what was a good decision and what not.
Reinforcement Learning uses the concept of a "Q-function" , which keeps track of how "good" it expects it to be to take a specific action 'a' from a specific location 's' . This is written as Q ( s , a ) . It also uses a "policy" , which determines the best action to take in a given state , and is written as π ( s ) . The robot must learn those functions while it navigates the maze . With each step , the functions are modified by a little bit , until eventually they give it the best strategy to solve the maze . ` ;
this . $lightbox . popup ( lightText , [ "continue" ] ) ;
} ,
} }
} ; render _latex ( ) ;
Ugo Finnendahl
