@ -1,31 +1,24 @@
Vue . component ( 'line-chart' , { // ----------------------------------------------------------------------------
extends : VueChartJs . Line , // ------------------------------- Utils --------------------------------------
mixins : [ VueChartJs . mixins . reactiveProp ] , // ----------------------------------------------------------------------------
props : [ 'options' ] ,
// mixins: [VueChartJs.mixins.reactiveData],
// props: ['options','labels', 'datasets'],
// watch: {
// 'labels': function(new_val) {
// this.chartData = {
// 'labels': new_val,
// 'datasets': this.datasets};
// },
// 'datasets': {
// deep:true,
// handler: function(new_val) {
// this.chartData = {
// 'labels': this.labels,
// 'datasets': new_val};
// }
// }
// },
mounted ( ) {
this . renderChart ( this . chartData , this . options ) ;
} ,
} ) function defer ( ) {
var res , rej ;
var promise = new Promise ( ( resolve , reject ) => {
res = resolve ;
rej = reject ;
} ) ;
promise . resolve = res ;
promise . reject = rej ;
return promise ;
}
var palette = [ '#00429d' , '#06449d' , '#0d469d' , '#12489d' , '#164a9d' , '#1a4c9c' , '#1d4e9c' , '#20509c' , '#23529c' , '#26549c' , '#28569b' , '#2b589b' , '#2d5a9b' , '#305c9b' , '#325e9a' , '#34609a' , '#37629a' , '#39649a' , '#3b6699' , '#3d6899' , '#3f6999' , '#416b98' , '#436d98' , '#456f97' , '#477197' , '#497397' , '#4b7596' , '#4d7796' , '#4f7995' , '#517b95' , '#537d94' , '#557f94' , '#578193' , '#598392' , '#5b8592' , '#5d8791' , '#5f8991' , '#618a90' , '#638c8f' , '#658e8f' , '#67908e' , '#69928d' , '#6b948d' , '#6d968c' , '#6f988b' , '#719a8b' , '#739c8a' , '#759e89' , '#77a088' , '#79a287' , '#7ba386' , '#7da586' , '#80a785' , '#82a984' , '#84ab83' , '#86ad82' , '#88af81' , '#8ab180' , '#8cb37f' , '#8fb57e' , '#91b77d' , '#93b87c' , '#95ba7a' , '#97bc79' , '#9abe78' , '#9cc077' , '#9ec276' , '#a1c474' , '#a3c673' , '#a5c872' , '#a8c970' , '#aacb6f' , '#accd6e' , '#afcf6c' , '#b1d16b' , '#b4d369' , '#b6d568' , '#b8d766' , '#bbd864' , '#bdda63' , '#c0dc61' , '#c2de5f' , '#c5e05d' , '#c8e25b' , '#cae459' , '#cde657' , '#cfe755' , '#d2e953' , '#d5eb50' , '#d7ed4e' , '#daef4b' , '#ddf049' , '#dff246' , '#e2f443' , '#e5f640' , '#e8f83c' , '#ebfa39' , '#edfb35' , '#f0fd31' , '#f3ff2c' ] // ----------------------------------------------------------------------------
// -------------------------------- Plot --------------------------------------
// ----------------------------------------------------------------------------
Array . prototype . simpleSMA = function ( N ) { Array . prototype . simpleSMA = function ( N ) {
return this . map ( return this . map (
@ -55,184 +48,163 @@ Array.prototype.max = function() {
} ) ; } ) ;
} ; } ;
app = new Vue ( { Vue . component ( 'line-chart' , {
el : '#app' , extends : VueChartJs . Line ,
components : { mixins : [ VueChartJs . mixins . reactiveProp ] ,
VueSlider : window [ 'vue-slider-component' ] props : [ 'options' ] ,
} , mounted ( ) {
data : { this . renderChart ( this . chartData , this . options ) ;
width : 0 ,
height : 0 ,
q _table : machine . q _table ,
maze : maze ,
state : {
x : 0 ,
y : 0
} , } ,
state _tween : new TimelineLite ( ) ,
score : machine . score , } )
score _history : machine . score _history ,
labels : [ ] , // ----------------------------------------------------------------------------
learning _rate : machine . lr , // --------------------------------- Map --------------------------------------
discount _factor : machine . df , // ----------------------------------------------------------------------------
epsilon : machine . epsilon ,
slider _config : { var MapBase = Vue . component ( 'MapBase' , {
min : 0 , props : [ 'machine' , 'maze' , 'config' ] ,
max : 1 , data : function ( ) {
duration : 0 , return {
interval : 0.01 , robot _image : null ,
tooltip : 'none' energy _image : null ,
} }
} , } ,
created ( ) { created ( ) {
// Resize handler
window . addEventListener ( 'resize' , this . handleResize )
this . handleResize ( ) ;
// State wrapper
var s = machine . state ;
var $this = this ; var $this = this ;
this . state = this . s2p ( s ) ; const robot _image = new window . Image ( ) ;
Object . defineProperty ( machine , 'state' , { robot _image . src = "img/robot.png" ;
get : function ( ) { robot _image . onload = ( ) => {
return this . _state $this . robot _image = robot _image ;
} ;
const energy _image = new window . Image ( ) ;
energy _image . src = "img/station.png" ;
energy _image . onload = ( ) => {
$this . energy _image = energy _image ;
} ;
} , } ,
set : function ( ne ) { computed : {
this . _state = ne ; main _config : function ( ) {
$this . handleState ( this . _state ) ; return {
offset : {
x : - ( this . config . width - this . base _size * this . maze . width ) / 2 ,
y : - ( this . config . height - this . base _size * this . maze . height ) / 2 ,
} }
} ) ;
machine . state = s ;
// Score wrapper
var s = machine . score ;
var $this = this ;
this . score = s ;
Object . defineProperty ( machine , 'score' , {
get : function ( ) {
return this . _score
} ,
set : function ( ne ) {
this . _score = ne ;
$this . score = ne
} }
} ) ;
machine . score = s ;
// Score history wrapper
var s = machine . score _history ;
var $this = this ;
this . score _history = s ;
Object . defineProperty ( machine , 'score_history' , {
get : function ( ) {
return this . _score _history
} , } ,
set : function ( ne ) { robot _config : function ( ) {
this . _score _history = ne ; return {
$this . score _history = ne height : this . base _size ,
width : this . base _size ,
x : this . center ,
y : this . center ,
image : this . robot _image ,
offset : {
x : this . base _size / 2 ,
y : this . base _size / 2 ,
}
} }
} ) ;
machine . score _history = s ;
} ,
destroyed ( ) {
window . removeEventListener ( 'resize' , this . handleResize )
} , } ,
computed : { energy _config : function ( ) {
datacollection : function ( ) {
return { return {
labels : Array . from ( Array ( this . score _history . length ) . keys ( ) ) , height : this . base _size ,
datasets : [ { width : this . base _size ,
label : 'Data One' , offset : {
backgroundColor : 'rgb(0,0,0,0)' , x : this . base _size / 2 ,
data : this . score _history , //.simpleSMA(Math.round(50)),
y : this . base _size / 2
fill : false ,
borderColor : 'rgb(255, 159, 64)' ,
pointRadius : 1 ,
} , } ,
// {
image : this . energy _image ,
// label: 'Data One',
// backgroundColor: 'rgb(0,0,0,0)',
// data: this.score_history.max(),
// fill: false,
// borderColor: 'rgb(64, 159, 255)',
// pointRadius: 1,
// },
]
} }
} , } ,
plot _options : function ( ) { strokeW : function ( ) {
var $this = this ; return this . base _size / 50 ;
return {
responsive : true ,
maintainAspectRatio : false ,
scales : {
xAxes : [ {
// type: 'linear',
ticks : {
maxTicksLimit : 8 ,
maxRotation : 0 ,
}
} ]
} , } ,
legend : { base _size : function ( ) {
display : false return Math . min ( this . config . height / this . maze . height , this . config . width / this . maze . width ) ;
} } ,
} ,
methods : {
get _tile _type : function ( state ) {
var pos = this . machine . s2p ( state ) ;
if ( pos . y > maze . height ) {
return null ;
} else if ( pos . x > maze . width ) {
return null ;
} else {
return maze . map [ pos . y ] [ pos . x ] ;
} }
} , } ,
stage _config : function ( ) { get _field _config : function ( state ) {
var pos = this . machine . s2p ( state ) ;
return { return {
width : this . width , x : this . base _size * pos . x + this . base _size / 2 ,
height : this . height , y : this . base _size * pos . y + this . base _size / 2 ,
} }
} , } ,
mini _map _config : function ( ) { get _tile _config : function ( t _type ) {
const layout = {
width : this . base _size ,
height : this . base _size ,
stroke : '#ddd' ,
strokeWidth : this . strokeW ,
offset : {
x : this . base _size / 2 ,
y : this . base _size / 2 ,
}
} ;
switch ( t _type ) {
case tile . regular :
return { return {
x : this . width / 2 - ( this . base _size * ( this . maze . width ) / 2 ) , ... layout ,
y : this . height / 2 - ( this . base _size * ( this . maze . height ) / 2 ) , fill : '#fff' ,
scale : { opacity : 1 ,
x : 1 ,
y : 1
} }
case tile . end :
return {
... layout ,
fill : '#0eb500' ,
opacity : 1 ,
} }
} , case tile . start :
local _layer : function ( ) {
return { return {
x : this . width / 2 , ... layout ,
y : this . height / 2 , fill : '#ff0008' ,
scale : { opacity : 1 ,
x : 2 ,
y : 2
} }
case tile . dangerous :
return {
... layout ,
fill : '#FF7B17' ,
opacity : 1 ,
} }
} , case tile . wall :
map _config : function ( ) {
return { return {
x : this . base _size * ( this . maze . width - this . state . x ) , ... layout ,
y : this . base _size * ( this . maze . height - this . state . y ) , fill : '#000000' ,
offset : { opacity : 1 ,
x : this . base _size * this . maze . width + this . base _size / 2 ,
y : this . base _size * this . maze . height + this . base _size / 2 ,
} }
} }
} , } ,
agent _config : function ( ) {
return {
sides : 5 ,
radius : this . base _size / 3 ,
fill : '#00D2FF' ,
stroke : 'black' ,
strokeWidth : this . strokeW ,
offset : {
x : - this . base _size / 2 ,
y : - this . base _size / 2
} , } ,
x : this . base _size * this . state . x , } )
y : this . base _size * this . state . y ,
//-----------------------------------------------------------------------------
var palette = [ '#d2000d' , '#d30512' , '#d40a17' , '#d50f1c' , '#d61420' , '#d71a25' , '#d71f2a' , '#d8242f' , '#d92934' , '#da2e39' , '#db333d' , '#dc3842' , '#dd3d47' , '#de424c' , '#df4751' , '#e04d56' , '#e0525a' , '#e1575f' , '#e25c64' , '#e36169' , '#e4666e' , '#e56b73' , '#e67077' , '#e7757c' , '#e87a81' , '#e98086' , '#e9858b' , '#ea8a90' , '#eb8f95' , '#ec9499' , '#ed999e' , '#ee9ea3' , '#efa3a8' , '#f0a8ad' , '#f1adb2' , '#f2b3b6' , '#f2b8bb' , '#f3bdc0' , '#f4c2c5' , '#f5c7ca' , '#f6cccf' , '#f7d1d3' , '#f8d6d8' , '#f9dbdd' , '#fae0e2' , '#fbe6e7' , '#fbebec' , '#fcf0f0' , '#fdf5f5' , '#fefafa' , '#ffffff' , '#fafcfa' , '#f5f9f5' , '#f0f6f0' , '#ebf3ec' , '#e6f1e7' , '#e1eee2' , '#dcebdd' , '#d7e8d8' , '#d3e5d3' , '#cee2cf' , '#c9dfca' , '#c4dcc5' , '#bfd9c0' , '#bad6bb' , '#b5d4b6' , '#b0d1b2' , '#abcead' , '#a6cba8' , '#a1c8a3' , '#9cc59e' , '#97c299' , '#92bf95' , '#8dbc90' , '#88b98b' , '#84b786' , '#7fb481' , '#7ab17c' , '#75ae77' , '#70ab73' , '#6ba86e' , '#66a569' , '#61a264' , '#5c9f5f' , '#579c5a' , '#529a56' , '#4d9751' , '#48944c' , '#439147' , '#3e8e42' , '#398b3d' , '#348839' , '#308534' , '#2b822f' , '#267f2a' , '#217d25' , '#1c7a20' , '#17771c' , '#127417' , '#0d7112' , '#086e0d' ]
Vue . component ( 'rl-map' , {
extends : MapBase ,
computed : {
robot _config : function ( ) {
return {
height : this . base _size ,
width : this . base _size ,
x : this . base _size * this . machine . state . x ,
y : this . base _size * this . machine . state . y ,
image : this . robot _image ,
} }
} , } ,
base _size : function ( ) {
return Math . min ( this . stage _config . height * 0.8 / this . maze . height , this . stage _config . width * 0.5 / this . maze . width ) ;
} ,
strokeW : function ( ) {
return this . base _size / 50 ;
} ,
extreme _q _values : function ( ) { extreme _q _values : function ( ) {
var max = - 10 * 30 ; var max = - 10 * 30 ;
var min = 10 * 30 ; var min = 10 * 30 ;
@ -245,70 +217,10 @@ app = new Vue({
} }
} }
} }
return { min : min , max : max } ; return { min : min , max : max } ;
} }
} , } ,
methods : { methods : {
s2p : function ( state ) {
return {
x : ( state % this . maze . width ) ,
y : Math . floor ( state / this . maze . width ) ,
}
} ,
p2s : function ( x , y ) {
return x + y * this . maze . width ;
} ,
handleResize : function ( ) {
this . width = window . innerWidth ;
this . height = window . innerHeight ;
} ,
handleState : function ( s ) {
if ( ! machine . running ) {
this . state _tween . to ( this . state , 0.2 , {
x : this . s2p ( s ) . x ,
y : this . s2p ( s ) . y
} ) ;
} else {
this . state = this . s2p ( s ) ;
}
// this.hidden_state = s;
} ,
get _grid _line _config : function ( idx , y = false ) {
var offset = this . strokeW / 2 ;
if ( y ) {
var points = [ - offset , Math . round ( idx * this . base _size ) , this . base _size * this . maze . width + offset , Math . round ( idx * this . base _size ) ] ;
} else {
var points = [ Math . round ( idx * this . base _size ) , - offset , Math . round ( idx * this . base _size ) , this . base _size * this . maze . height + offset ] ;
}
return {
points : points ,
stroke : '#ddd' ,
strokeWidth : this . strokeW ,
}
} ,
get _tile _type : function ( state ) {
var pos = this . s2p ( state ) ;
if ( pos . y > maze . height ) {
return null ;
} else if ( pos . x > maze . width ) {
return null ;
} else {
return maze . map [ pos . y ] [ pos . x ] ;
}
} ,
in _plus : function ( pos1 , pos2 ) {
if ( Math . abs ( pos1 . x - pos2 . x ) + Math . abs ( pos1 . y - pos2 . y ) < 2 ) {
return true ;
}
return false ;
} ,
get _field _config : function ( state ) {
var pos = this . s2p ( state ) ;
return {
x : this . base _size * pos . x + this . base _size / 2 ,
y : this . base _size * pos . y + this . base _size / 2 ,
}
} ,
get _q _text _config : function ( val , i ) { get _q _text _config : function ( val , i ) {
var off , key ; var off , key ;
switch ( i ) { switch ( i ) {
@ -348,13 +260,13 @@ app = new Vue({
fontSize : this . base _size / 7 , fontSize : this . base _size / 7 ,
fontFamily : 'Calibri' , fontFamily : 'Calibri' ,
fill : 'black' , fill : 'black' ,
text : + val [ key ] . toFixed ( 2 ) + '' , text : + val [ key ] . toPrecision ( 3 ) + '' ,
width : this . base _size - 5 , width : this . base _size - 20 ,
height : this . base _size - 5 , height : this . base _size - 34 ,
... off , ... off ,
offset : { offset : {
x : ( this . base _size - 5 ) / 2 , x : ( this . base _size - 20 ) / 2 ,
y : ( this . base _size - 5 ) / 2 , y : ( this . base _size - 34 ) / 2 ,
} }
} }
} , } ,
@ -375,109 +287,478 @@ app = new Vue({
break ; break ;
} }
var $this = this ; var $this = this ;
var norma _value = ( value - this . extreme _q _values . min ) / ( ( this . extreme _q _values . max - this . extreme _q _values . min ) || 1 ) ; var norma _value = value > 0 ? ( value + 1000 ) / ( 2000 ) : ( value + 30 ) / 60 ;
return { return {
sceneFunc : function ( context , shape ) { sceneFunc : function ( context , shape ) {
context . beginPath ( ) ; context . beginPath ( ) ;
context . moveTo ( 0 , 0 ) ; var width = $this . base _size / 5 ;
context . lineTo ( $this . base _size / 2 , $this . base _size / 2 ) ; var arrow _w = $this . base _size / 2 ;
context . lineTo ( $this . base _size / 2 , - $this . base _size / 2 ) ; var stumpf = $this . base _size / 6 ;
context . lineTo ( 0 , 0 ) ; var arrow _l = $this . base _size / 5 ;
context . moveTo ( $this . base _size / 2 - stumpf - arrow _l , width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , arrow _w / 2 ) ;
context . lineTo ( $this . base _size / 2 - 2 , 0 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , - arrow _w / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf , - width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf - arrow _l , - width / 2 ) ;
context . lineTo ( $this . base _size / 2 - stumpf - arrow _l , width / 2 ) ;
context . closePath ( ) ; context . closePath ( ) ;
// (!) Konva specific method, it is very important
// (!) Konva specific method, it is very important
context . fillStrokeShape ( shape ) ; context . fillStrokeShape ( shape ) ;
} , } ,
fill : palette [ Math . round ( norma _value * 99 ) ] , fill : palette [ Math . round ( norma _value * 100 ) ] ,
stroke : 'black' , stroke : 'black' ,
strokeWidth : 0 , strokeWidth : 1 ,
rotation : rot , rotation : rot ,
} }
} , } ,
get _tile _config : function ( i , t _type , local = false ) { } ,
var pos = this . s2p ( i ) ; template :
var over = { } ; ` <v-stage ref="stage" :config="config">
< v - layer ref = "map_layer" : config = "main_config" >
< v - group ref = "map_group" >
< v - group : key = "'tile'+idx" v - for = "(t_type, idx) in maze.map.flat()" : config = "get_field_config(idx)" >
< v - rect : config = "get_tile_config(t_type)" > < / v - r e c t >
< v - image : config = "energy_config" v - if = "t_type==8" > < / v - i m a g e >
< / v - g r o u p >
< v - group : key = "'qgroup'+idx" v - for = "(action, idx) in machine.q_table" : config = "get_field_config(idx)" >
< v - shape : key = "'qvalshape'+idx+key" v - for = "(value, key) in action" : config = "get_triangle_config(value, key)" > < / v - s h a p e >
< v - text : key = "'qval'+idx+i" v - for = "i in 4" : config = "get_q_text_config(action,i)" > < / v - t e x t >
< / v - g r o u p >
< v - image : config = "robot_config" > < / v - i m a g e >
< / v - g r o u p >
< / v - l a y e r >
< / v - s t a g e > `
} )
// not in plus
// ----------------------------------------------------------------------------
if ( local ) { // -------------------------------- Local -------------------------------------
if ( ! this . in _plus ( this . s2p ( i ) , { // ----------------------------------------------------------------------------
x : Math . round ( this . state . x ) ,
y : Math . round ( this . state . y ) Vue . component ( 'rl-local' , {
} ) ) { extends : MapBase ,
over = { computed : {
opacity : 0 , main _config : function ( ) {
fill : "#eee" return {
} ; offset : {
} else if ( i != this . p2s ( Math . round ( this . state . x ) , Math . round ( this . state . y ) ) ) { x : - ( this . config . width - this . base _size * 3 ) / 2 ,
over = { y : - ( this . config . height - this . base _size * 3 ) / 2 ,
opacity : 1 ,
fill : "#eee"
} ;
} }
} }
const layout = { } ,
x : this . base _size * pos . x , local _config : function ( ) {
y : this . base _size * pos . y , return {
x : - ( this . machine . state . x ) * this . base _size ,
y : - ( this . machine . state . y ) * this . base _size ,
offset : {
x : - this . base _size ,
y : - this . base _size ,
}
}
} ,
base _size : function ( ) {
return Math . min ( this . config . height / 3 , this . config . width / 3 ) ;
} ,
center : function ( ) {
return 3 * this . base _size / 2 ;
} ,
local _area : function ( ) {
const x = Math . round ( this . machine . state . x ) ;
const y = Math . round ( this . machine . state . y ) ;
let arr = [ [ x , y - 1 ] , [ x + 1 , y ] , [ x , y + 1 ] , [ x - 1 , y ] , [ x , y ] ] ;
return arr . filter ( ( p ) => p [ 0 ] < this . maze . width && p [ 1 ] < this . maze . height &&
p [ 0 ] >= 0 && p [ 1 ] >= 0 )
. map ( ( p ) => [ this . maze . map [ p [ 1 ] ] [ p [ 0 ] ] , p [ 1 ] * this . maze . width + p [ 0 ] ] ) ;
} ,
} ,
methods : {
end : function ( pos ) {
return this . maze . get _states ( tile . end ) . indexOf ( pos ) >= 0 ;
} ,
id _to _dir : function ( id ) {
switch ( id ) {
case 0 :
return dir . UP ;
case 1 :
return dir . RIGHT ;
case 2 :
return dir . DOWN ;
case 3 :
return dir . LEFT ;
default :
return undefined ;
}
} ,
handleMouseEnter ( e ) {
const stage = e . target . getStage ( ) ;
stage . container ( ) . style . cursor = "pointer" ;
} ,
handleMouseLeave ( e ) {
const stage = e . target . getStage ( ) ;
stage . container ( ) . style . cursor = "default" ;
} ,
get _local _tile _config : function ( i , t _type ) {
// var pos = this.s2p(i);
// in plus
var over = { } ;
if ( i != this . machine . p2s ( Math . round ( this . machine . state . x ) , Math . round ( this . machine . state . y ) ) &&
t _type != tile . wall ) {
over = {
width : this . base _size , width : this . base _size ,
height : this . base _size , height : this . base _size ,
stroke : '#ddd' , stroke : '#ddd' ,
strokeWidth : this . strokeW , strokeWidth : this . strokeW ,
} ; offset : {
switch ( t _type ) { x : this . base _size / 2 ,
case tile . regular : y : this . base _size / 2 ,
return { } ,
... layout ,
fill : '#fff' ,
opacity : 1 , opacity : 1 ,
... over , fill : "#eee" ,
} }
case tile . end : }
return over ;
} ,
} ,
template :
` <v-stage ref="stage" :config="config">
< v - layer ref = "map_layer" : config = "main_config" >
< v - group ref = "map_group" : config = "local_config" >
< v - group : key = "pair[1]" v - for = "(pair, idx) in local_area" : config = "get_field_config(pair[1])" >
< v - rect : config = "get_tile_config(pair[0])" > < / v - r e c t >
< v - image : config = "energy_config" v - if = "end(pair[1])" > < / v - i m a g e >
< v - rect : config = "get_local_tile_config(pair[1], pair[0])" @ click = "id_to_dir(idx) && machine.object.step(id_to_dir(idx))" @ mouseenter = "handleMouseEnter" @ mouseleave = "handleMouseLeave" > < / v - r e c t >
< / v - g r o u p >
< / v - g r o u p >
< v - image : config = "robot_config" > < / v - i m a g e >
< / v - l a y e r >
< / v - s t a g e > `
} )
Vue . component ( 'navi-gation' , {
props : [ "options" ] ,
template : `
< nav class = "navi" >
< button v - for = "(item, key) in options" v - on : click = "item" > { { key } } < / b u t t o n >
< / n a v > `
} ) ;
// ----------------------------------------------------------------------------
// ------------------------------ lightbox ------------------------------------
// ----------------------------------------------------------------------------
var light _box = {
data : {
content : "" ,
options : [ ] ,
active : false ,
} ,
methods : {
close : function ( ) {
this . active = false ;
} ,
popup : function ( content , options ) {
this . content = content ;
var answer = defer ( ) ;
var $this = this ;
this . options = options . reduce ( ( old , opt ) => {
old [ opt ] = function ( ) {
$this . active = false ;
answer . resolve ( opt ) ;
}
return old
} , { } ) ;
this . active = true ;
return answer ;
}
} ,
template : `
< div class = "lightbox" v - bind : class = "{ active: active }" > { { content } }
< div class = "options" >
< button : key = "key" v - for = "(item, key) in options" v - on : click = "item" > { { key } } < / b u t t o n >
< / d i v >
< / d i v > `
}
const PopupLibrary = {
install ( Vue , options = { } ) {
const root = new Vue ( light _box )
// Mount root Vue instance on new div element added to body
root . $mount ( document . body . appendChild ( document . createElement ( 'div' ) ) )
Vue . prototype . $lightbox = root ;
}
}
window . Vue . use ( PopupLibrary )
// ----------------------------------------------------------------------------
// -------------------------------- Main --------------------------------------
// ----------------------------------------------------------------------------
function makeMachineReactive ( th , machine ) {
var $this = th ;
$this . machine . s2p = function ( state ) {
return { return {
... layout , x : ( state % $this . maze . width ) ,
fill : '#0eb500' , y : Math . floor ( state / $this . maze . width ) ,
opacity : 1 ,
... over ,
} }
case tile . start : } ;
$this . machine . p2s = function ( x , y ) {
return x + y * $this . maze . width ;
} ;
// Score wrapper
var s = machine . score ;
$this . machine . score = s ;
Object . defineProperty ( machine , 'score' , {
get : function ( ) {
return this . _score
} ,
set : function ( ne ) {
this . _score = ne ;
$this . machine . score = ne
}
} ) ;
machine . score = s ;
// Score history wrapper
var s = machine . score _history ;
$this . machine . score _history = s ;
Object . defineProperty ( machine , 'score_history' , {
get : function ( ) {
return this . _score _history
} ,
set : function ( ne ) {
this . _score _history = ne ;
$this . machine . score _history = ne
}
} ) ;
machine . score _history = s ;
// State wrapper
var s = machine . state ;
$this . machine . state = $this . machine . s2p ( s ) ;
Object . defineProperty ( machine , 'state' , {
get : function ( ) {
return this . _state
} ,
set : function ( ne ) {
this . _state = ne ;
$this . handleState ( this . _state ) ;
}
} ) ;
machine . state = s ;
$this . machine . object . setCallback ( $this . onNewEpisode ) ;
}
app = new Vue ( {
el : '#app' ,
components : {
VueSlider : window [ 'vue-slider-component' ] ,
} ,
data : {
state : null ,
maze : maze ,
machine : {
object : machine ,
q _table : machine . q _table ,
state : {
x : 0 ,
y : 0 ,
} ,
state _tween : new TimelineLite ( ) ,
learning _rate : machine . lr ,
discount _factor : machine . df ,
epsilon : machine . epsilon ,
score : machine . score ,
score _history : machine . score _history ,
s2p : null ,
p2s : null ,
} ,
width : 0 ,
height : 0 ,
components : [ ] ,
navigation : { } ,
} ,
created ( ) {
// Resize handler
window . addEventListener ( 'resize' , this . handleResize )
this . handleResize ( ) ;
makeMachineReactive ( this , machine ) ;
this . state = "init" ;
} ,
destroyed ( ) {
window . removeEventListener ( 'resize' , this . handleResize )
} ,
computed : {
stage _config : function ( ) {
return { return {
... layout , x : 0 ,
fill : '#ff0008' , y : 0 ,
opacity : 1 , width : this . width * 0.5 ,
... over , height : this . height * 0.8 ,
} }
case tile . dangerous : } ,
slider _config : function ( ) {
return { return {
... layout , min : 0 ,
fill : '#FF7B17' , max : 1 ,
opacity : 1 , duration : 0 ,
... over , interval : 0.01 ,
tooltip : 'none'
} }
case tile . wall : } ,
datacollection : function ( ) {
return { return {
... layout , labels : Array . from ( Array ( this . machine . score _history . length ) . keys ( ) ) ,
fill : '#000000' , datasets : [ {
opacity : 1 , label : 'Data One' ,
... over , backgroundColor : 'rgb(0,0,0,0)' ,
data : this . machine . score _history , //.simpleSMA(Math.round(50)),
fill : false ,
borderColor : 'rgb(255, 159, 64)' ,
pointRadius : 1 ,
} ,
// {
// label: 'Data One',
// backgroundColor: 'rgb(0,0,0,0)',
// data: this.score_history.max(),
// fill: false,
// borderColor: 'rgb(64, 159, 255)',
// pointRadius: 1,
// },
]
} }
} ,
plot _options : function ( ) {
var $this = this ;
return {
responsive : true ,
maintainAspectRatio : false ,
scales : {
xAxes : [ {
// type: 'linear',
ticks : {
maxTicksLimit : 8 ,
maxRotation : 0 ,
} }
} ]
} ,
legend : {
display : false
}
}
} ,
} ,
methods : {
onEnterState : function ( ) { } ,
onLeaveState : function ( ) { } ,
handleState : function ( s ) {
if ( ! this . machine . object . running ) {
this . machine . state _tween . to ( this . machine . state , 0.2 , {
x : this . machine . s2p ( s ) . x ,
y : this . machine . s2p ( s ) . y
} ) ;
} else {
this . machine . state = this . machine . s2p ( s ) ;
}
} ,
handleResize : function ( ) {
this . width = window . innerWidth ;
this . height = window . innerHeight ;
} ,
isActive : function ( what ) {
return this . components . indexOf ( what ) >= 0 ;
} ,
changeState : function ( state ) {
this . components = [ ] ;
this . navigation = { } ;
this . onEnterState = function ( ) { } ;
this . onLeaveState = function ( ) { } ;
this . state = state ;
} ,
onNewEpisode : function ( result ) {
var text ;
if ( result == "failed" ) {
text = "Out of battery. The robot will be resetted." ;
} else if ( result == "success" ) {
text = "You reached the goal. The robot will be resetted." ;
}
return this . $lightbox . popup ( text , [ "ok" ] ) ;
} }
} , } ,
watch : { watch : {
learning _rate : function ( new _val ) { 'machine.learning_rate' : function ( new _val ) {
machine . lr = new _val ; machine . lr = new _val ;
render _latex ( ) ; renderL atex ( ) ;
} , } ,
discount _factor : function ( new _val ) { 'machine.discount_factor' : function ( new _val ) {
machine . df = new _val ; machine . df = new _val ;
render _latex ( ) ; renderL atex ( ) ;
} , } ,
epsilon : function ( new _val ) { 'machine.epsilon' : function ( new _val ) {
machine . epsilon = new _val ; machine . epsilon = new _val ;
} } ,
state : function ( state ) {
this . onLeaveState ( ) ;
Object . assign ( this , StateMgr [ state ] ) ;
this . onEnterState ( ) ;
} ,
} }
} ) } )
function render _latex ( ) { function renderL atex ( ) {
// (1-lr) * Q[state, action] + lr * (reward + gamma * np.max(Q[new_state, :])
// (1-lr) * Q[state, action] + lr * (reward + gamma * np.max(Q[new_state, :])
katex . render ( ` Q(s,a) \\ leftarrow ${ ( 1 - machine . lr ) . toFixed ( 2 ) } Q(s,a)+ ${ machine . lr . toFixed ( 2 ) } (reward + ${ machine . df . toFixed ( 2 ) } * \\ max_a(Q(s', a)) ` , document . getElementById ( 'test' ) , { displayMode : true , } ) ; katex . render ( ` Q(s,a) \\ leftarrow ${ ( 1 - machine . lr ) . toFixed ( 2 ) } Q(s,a)+ ${ machine . lr . toFixed ( 2 ) } (reward + ${ machine . df . toFixed ( 2 ) } \\ max_{ a'} (Q(s_{new}, a ')) ` , document . getElementById ( 'formula ' ) , { displayMode : true , } ) ;
} }
render _latex ( ) ; renderLatex ( ) ;
// ----------------------------------------------------------------------------
// ------------------------------ StateMgr ------------------------------------
// ----------------------------------------------------------------------------
var StateMgr = {
init : {
onEnterState : function ( ) {
var lightText = ` Reinforcement learning (RL) is an area of machine learning concerned with how software agents ought to take actions in an environment so as to maximize some notion of cumulative reward. Reinforcement learning is one of three basic machine learning paradigms, alongside supervised learning and unsupervised learning. (wikipedia)
This exhibit explains how a robot can learn to navigate through a maze in order to reach its destination , before running out of power . At first the robot knows nothing , and learns from each new action ( movement ) and state ( location reached ) . Slowly it starts to develop an understanding of the maze that will allow it to reach the charging station before it runs out of power . Eventually , it should learn to avoid any detour and reach the charging station in the optimal number of steps . `
this . $lightbox . popup ( lightText , [ "next" ] ) . then ( ( r ) => this . changeState ( "local" ) ) ;
} ,
} ,
local : {
components : [ "local" , "navi" , "score" ] ,
navigation : {
"reset robot" : ( ) => machine . reset _machine ( ) ,
"continue" : null ,
} ,
onEnterState : function ( ) {
this . navigation . continue = ( ) => this . changeState ( "global" ) ;
var lightText = "But there is a problem! The robot cannot see the whole maze, it only knows where it is and in which direction it can move. Can you reach the charging station in those conditions? Use the arrows to move" ;
this . $lightbox . popup ( lightText , [ "next" ] ) ;
} ,
} ,
global : {
components : [ "global" , "sliders" , "plot" , "navi" , "score" ] ,
navigation : {
"run 1 episode!" : ( ) => machine . run ( 1 ) ,
"run 100 episodes!" : ( ) => machine . run ( 100 ) ,
"auto step!" : ( ) => machine . auto _step ( ) ,
"greedy step!" : ( ) => machine . greedy _step ( ) ,
"reset machine" : ( ) => machine . reset _machine ( ) ,
} ,
onEnterState : function ( ) {
var lightText = ` As a human, you keep track of where you are and how you got there without thinking, which helps you think about what actions you should take next to reach your destination. And you can also just look around! How can then the robot 'think' of the maze, to know which action is the best at every moment? And how can it learn that? It must somehow keep track of where it is, and remember how good or bad was each action at each place in the maze, try new things, and update it's "mental image" of what was a good decision and what not.
Reinforcement Learning uses the concept of a "Q-function" , which keeps track of how "good" it expects it to be to take a specific action 'a' from a specific location 's' . This is written as Q ( s , a ) . It also uses a "policy" , which determines the best action to take in a given state , and is written as π ( s ) . The robot must learn those functions while it navigates the maze . With each step , the functions are modified by a little bit , until eventually they give it the best strategy to solve the maze . ` ;
this . $lightbox . popup ( lightText , [ "continue" ] ) ;
} ,
}
} ;
Ugo Finnendahl
