Return to Question

from pygame import * from pygame.locals import * import sys from time import sleep import numpy as np import random from time import sleep import tensorflow as tf #Screen Setup disp_x, disp_y = 1500, 1000 arena_x, arena_y = 800, 800 border = 4; border_2 = 1 #Color Setup white = (255, 255, 255); aqua= (0, 200, 200) red = (255, 0, 0); green = (0, 255, 0) blue = (0, 0, 255); black = (0, 0, 0) #Initialize character positions init_character_a_state = [disp_x/2 - arena_x/2 + 50, disp_y/2 - arena_y/2 + 50] init_character_b_state = [disp_x/2 + arena_x/2 - 50, disp_y/2 + arena_y/2 - 50] #Setup character dimentions character_radius = 30 character_move_speed = 20 #Initialize character stats character_init_health = 100 #initialize bullet stats bullet_speed = 50 bullet_damage = 10 bullet_radius = 7 bullet_a_pos = list(init_character_a_state); bullet_b_pos = list(init_character_b_state) bullet_a_fire = False; bullet_b_fire = False #The Neural Network input_layer = tf.placeholder(shape=[1,8],dtype=tf.float32) weight_1 = tf.Variable(tf.random_uniform([8,9],0,0.1)) #The calculations, loss function and the update model Q = tf.matmul(input_layer, weight_1) predict = tf.argmax(Q, 1) next_Q = tf.placeholder(shape=[1,9],dtype=tf.float32) loss = tf.reduce_sum(tf.square(next_Q - Q)) trainer = tf.train.GradientDescentOptimizer(learning_rate=0.0001) updateModel = trainer.minimize(loss) initialize = tf.global_variables_initializer()    jList = [] rList = [] init() font.init() myfont = font.SysFont('Comic Sans MS', 15) myfont2 = font.SysFont('Comic Sans MS', 150) myfont3 = font.SysFont('Gothic', 30) disp = display.set_mode((disp_x, disp_y), 0, 32) bullet_ob = -100 #CHARACTER/BULLET PARAMETERS bot_bullet_x = bot_bullet_y = bullet_ob agent_bullet_x = agent_bullet_y = bullet_ob last_bot_bullet_x = last_bot_bullet_y = bullet_ob last_agent_bullet_x = last_agent_bullet_y = bullet_ob agent_bullet_fire = bot_bullet_fire = bool() agent_bullet_direction_x = agent_bullet_direction_y = int() bot_bullet_direction_x = bot_bullet_direction_y = int() agent_x = agent_y = int() bot_x = bot_y = int() agent_hp = bot_hp = int() def param_init(): """Initializes parameters""" global bot_bullet_x, bot_bullet_y, agent_bullet_x, agent_bullet_y, agent_bullet_fire, \  bot_bullet_fire, agent_bullet_direction_x, agent_bullet_direction_y, bot_bullet_direction_x, \  bot_bullet_direction_y, agent_x, agent_y, bot_x, bot_y, agent_hp, bot_hp  agent_bullet_x = agent_bullet_y = bullet_ob bot_bullet_x = bot_bullet_y = bullet_ob last_agent_bullet_x = last_agent_bullet_y = bullet_ob last_bot_bullet_x = last_bot_bullet_y = bullet_ob agent_bullet_fire = bot_bullet_fire = False agent_bullet_direction_x = 0; agent_bullet_direction_y = 0 bot_bullet_direction_x = 0; bot_bullet_direction_y = 0  agent_x = list(init_character_a_state)[0]; agent_y = list(init_character_a_state)[1] bot_x = list(init_character_b_state)[0]; bot_y = list(init_character_b_state)[1] agent_hp = bot_hp = character_init_health def screen_blit(): global disp, disp_x, disp_y, arena_x, arena_y, border, border_2, agent_bullet_x, \ agent_bullet_y, bullet_radius, bot_bullet_x, bot_bullet_y, character_radius, agent_x, \ agent_y, bot_x, bot_y, character_init_health, agent_hp, bot_hp, red, blue, aqua, green, black disp.fill(aqua) draw.rect(disp, black, (disp_x / 2 - arena_x / 2 - border, disp_y / 2 - arena_y / 2 - border, arena_x + border * 2, arena_y + border * 2)) draw.rect(disp, green, (disp_x / 2 - arena_x / 2, disp_y / 2 - arena_y / 2, arena_x, arena_y)) draw.circle(disp, black, [agent_bullet_x, agent_bullet_y], bullet_radius)   draw.circle(disp, black, [bot_bullet_x, bot_bullet_y], bullet_radius)  draw.circle(disp, black, (agent_x, agent_y),  character_radius + border_2)  draw.circle(disp, red, (agent_x, agent_y), character_radius)  draw.circle(disp, black, (bot_x, bot_y), character_radius + border_2) draw.circle(disp, blue, (bot_x, bot_y), character_radius) draw.rect(disp, red, (disp_x / 2 - 200, disp_y / 2 + arena_y / 2 + border + 1, float(agent_hp) / float(character_init_health) * 100, 14)) draw.rect(disp, blue, (disp_x / 2 + 200, disp_y / 2 + arena_y / 2 + border + 1, float(bot_hp) / float(character_init_health) * 100, 14)) def bot_take_action(): global agent_x, agent_y, bot_x, bot_y, character_radius, bot_action, border if agent_x - character_radius - border <= bot_x <= agent_x + character_radius + border: if random.randint(0, 100) > 5: if agent_y <= bot_y:   bot_action = 1 else: bot_action = 3   else:  bot_action = 9  elif agent_y - character_radius <= bot_y <= agent_y + character_radius:  if random.randint(0, 100) > 5:  if agent_x <= bot_x: bot_action = 4 else: bot_action = 2 else: bot_action = 9  else: if random.randint(0, 100) > 5:   x_dist = abs(bot_x - agent_x); y_dist = abs(bot_y - agent_y) if x_dist >= y_dist: if bot_x - agent_x <= 0:  bot_action = 6  else:   bot_action = 8 else: if bot_y - agent_y <= 0: bot_action = 7 else: bot_action = 5 else: bot_action = random.randint(1, 9) def bullet_hit_detector(player): global bot_bullet_x, bot_bullet_y, last_bot_bullet_x, last_bot_bullet_y, agent_x, agent_y, last_agent_bullet_x, last_agent_bullet_y, character_radius, border, bullet_radius if player == "bot": if bot_bullet_x == last_bot_bullet_x: if agent_x - character_radius - border < bot_bullet_x + bullet_radius < agent_x + character_radius + border or \ agent_x - character_radius - border < bot_bullet_x - bullet_radius < agent_x + character_radius + border:  #If the current state of the bullet is touching/inside the agent: if agent_y - character_radius - border < bot_bullet_y + bullet_radius < agent_y + character_radius or \ agent_y - character_radius < bot_bullet_y - bullet_radius < agent_y + character_radius + border: return True #If the bullet "passed through" the character from the last turn: elif (last_bot_bullet_y - bullet_radius > agent_y + character_radius + border and agent_y - character_radius - border > bot_bullet_y + bullet_radius) \  or (bot_bullet_y - bullet_radius > agent_y + character_radius + border and agent_y - character_radius - border > last_bot_bullet_y + bullet_radius): return True else: return False elif bot_bullet_y == last_bot_bullet_y: if agent_y - character_radius - border < bot_bullet_y - bullet_radius < agent_y + character_radius + border or \  agent_y - character_radius - border < bot_bullet_y + bullet_radius < agent_y + character_radius + border: #If the current state of the bullet is touching/inside the agent: if agent_x - character_radius - border < bot_bullet_x + bullet_radius < agent_x + character_radius or \ agent_x - character_radius < bot_bullet_x - bullet_radius < agent_x + character_radius + border: return True #If the bullet "passed through" the character from the last turn: elif (last_bot_bullet_x - bullet_radius > agent_x + character_radius + border and agent_x - character_radius - border > bot_bullet_x + bullet_radius) \ or (bot_bullet_x - bullet_radius > agent_x + character_radius + border and agent_x - character_radius - border > last_bot_bullet_x + bullet_radius):   return True   else:   return False else: if agent_bullet_x == last_agent_bullet_x: if bot_x - character_radius - border < agent_bullet_x + bullet_radius < bot_x + character_radius + border or \  bot_x - character_radius - border < agent_bullet_x - bullet_radius < bot_x + character_radius + border:   #If the current state of the bullet is touching/inside the agent: if bot_y - character_radius - border < agent_bullet_y + bullet_radius < bot_y + character_radius or \  bot_y - character_radius < agent_bullet_y - bullet_radius < bot_y + character_radius + border:  return True #If the bullet "passed through" the character from the last turn: elif (last_agent_bullet_y - bullet_radius > bot_y + character_radius + border and bot_y - character_radius - border > agent_bullet_y + bullet_radius) \  or (agent_bullet_y - bullet_radius > bot_y + character_radius + border and bot_y - character_radius - border > last_agent_bullet_y + bullet_radius):   return True else:  return False  elif agent_bullet_y == last_agent_bullet_y: if bot_y - character_radius - border < agent_bullet_y - bullet_radius < bot_y + character_radius + border or \   bot_y - character_radius - border < agent_bullet_y + bullet_radius < bot_y + character_radius + border:  #If the current state of the bullet is touching/inside the agent: if bot_x - character_radius - border < agent_bullet_x + bullet_radius < bot_x + character_radius or \ bot_x - character_radius < agent_bullet_x - bullet_radius < bot_x + character_radius + border: return True #If the bullet "passed through" the character from the last turn: elif (last_agent_bullet_x - bullet_radius > bot_x + character_radius + border and bot_x - character_radius - border > agent_bullet_x + bullet_radius) \ or (agent_bullet_x - bullet_radius > bot_x + character_radius + border and bot_x - character_radius - border > last_agent_bullet_x + bullet_radius): return True else: return False  def mapping(maximum, number): return int(abs(number * maximum) / (maximum/10)) def action(agent_action, bot_action): global bot_bullet_x, bot_bullet_y, agent_bullet_x, agent_bullet_y, agent_bullet_fire, \ bot_bullet_fire, agent_bullet_direction_x, agent_bullet_direction_y, bot_bullet_direction_x, \ bot_bullet_direction_y, agent_x, agent_y, bot_x, bot_y, agent_hp, bot_hp, last_agent_bullet_x, last_agent_bullet_y, last_bot_bullet_x, last_bot_bullet_y reward = 0; cont = True; successful = False; winner = "" if 1 <= bot_action <= 4 and bot_bullet_fire == False: bot_bullet_fire = True if bot_action == 1: bot_bullet_direction_x = 0; bot_bullet_direction_y = -bullet_speed elif bot_action == 2: bot_bullet_direction_x = bullet_speed; bot_bullet_direction_y = 0 elif bot_action == 3: bot_bullet_direction_x = 0; bot_bullet_direction_y = bullet_speed elif bot_action == 4: bot_bullet_direction_x = -bullet_speed; bot_bullet_direction_y = 0 bot_bullet_x = bot_x + bot_bullet_direction_x; bot_bullet_y = bot_y + bot_bullet_direction_y elif 5 <= bot_action <= 8: if bot_action == 5: bot_y -= character_move_speed if bot_y <= disp_y/2 - arena_y/2 + character_radius + 1: bot_y = disp_y/2 - arena_y/2 + character_radius + 1 elif bot_action == 6: bot_x += character_move_speed if bot_x >= disp_x/2 + arena_x/2 - character_radius - 1: bot_x = disp_x/2 + arena_x/2 - character_radius - 1 elif bot_action == 7: bot_y += character_move_speed if bot_y >= disp_y/2 + arena_y/2 - character_radius - 1: bot_y = disp_y/2 + arena_y/2 - character_radius - 1 elif bot_action == 8: bot_x -= character_move_speed if bot_x <= disp_x/2 - arena_x/2 + character_radius + 1: bot_x = disp_x/2 - arena_x/2 + character_radius + 1 if bot_bullet_fire == True: last_bot_bullet_x = bot_bullet_x; last_bot_bullet_y = bot_bullet_y bot_bullet_x += bot_bullet_direction_x; bot_bullet_y += bot_bullet_direction_y  if bullet_hit_detector("bot"): print "Agent Got Hit!" agent_hp -= bullet_damage reward = -50 bot_bullet_fire = False bot_bullet_direction_x = 0; bot_bullet_direction_y = 0   bot_bullet_x = bot_bullet_y = bullet_ob; last_bot_bullet_x = last_bot_bullet_y = bullet_ob if agent_hp <= 0: cont = False winner = "Bot"  elif bot_bullet_x + bullet_radius >= disp_x/2 + arena_x/2 or bot_bullet_x - bullet_radius <= disp_x/2 - arena_x/2 or \ bot_bullet_y + bullet_radius >= disp_y/2 + arena_y/2 or bot_bullet_y - bullet_radius <= disp_y/2 - arena_y/2: bot_bullet_fire = False bot_bullet_direction_x = 0; bot_bullet_direction_y = 0 bot_bullet_x = bot_bullet_y = bullet_ob; last_bot_bullet_x = last_bot_bullet_y = bullet_ob  if 1 <= agent_action <= 4: if agent_bullet_fire == False: agent_bullet_fire = True   if agent_action == 1:   if agent_y - character_radius - border > disp_y/2 - arena_y/2:   agent_bullet_direction_x = 0; agent_bullet_direction_y = -bullet_speed  reward = 10  else:   reward = -25 agent_bullet_x = agent_bullet_y = bullet_ob agent_bullet_fire = False   elif agent_action == 2:   if agent_x + character_radius + border < disp_x/2 + arena_x/2:   agent_bullet_direction_x = bullet_speed; agent_bullet_direction_y = 0  reward = 10  else:   reward = -25  agent_bullet_x = agent_bullet_y = bullet_ob   agent_bullet_fire = False   elif agent_action == 3:   if agent_y + character_radius + border < disp_y/2 + arena_y/2:   agent_bullet_direction_x = 0; agent_bullet_direction_y = bullet_speed  reward = 10  else:   reward = -25  agent_bullet_x = agent_bullet_y = bullet_ob   agent_bullet_fire = False   elif agent_action == 4:   if agent_x - character_radius - border > disp_x/2 - arena_x/2:   agent_bullet_direction_x = -bullet_speed; agent_bullet_direction_y = 0  reward = 10  else:  reward = -25  agent_bullet_x = agent_bullet_y = bullet_ob   agent_bullet_fire = False  if agent_bullet_fire == True: agent_bullet_x = agent_x + agent_bullet_direction_x; agent_bullet_y = agent_y + agent_bullet_direction_y   last_agent_bullet_x = agent_bullet_x; last_agent_bullet_y = agent_bullet_y else:  reward = -20 elif 5 <= agent_action <= 8: if agent_action == 5: agent_y -= character_move_speed if agent_y - character_radius - border <= disp_y/2 - arena_y/2: agent_y = disp_y/2 - arena_y/2 + character_radius + border reward = -5 else: reward = 5 elif agent_action == 6: agent_x += character_move_speed if agent_x + character_radius + border >= disp_x/2 + arena_x/2: agent_x = disp_x/2 + arena_x/2 - character_radius - border reward = -5 else: reward = 5 elif agent_action == 7: agent_y += character_move_speed if agent_y + character_radius + border >= disp_y/2 + arena_y/2: agent_y = disp_y/2 + arena_y/2 - character_radius - border reward = -5 else: reward = 5 elif agent_action == 8: agent_x -= character_move_speed if agent_x - character_radius - border <= disp_x/2 - arena_x/2: agent_x = disp_x/2 - arena_x/2 + character_radius + border reward = -5 else:  reward = 5  if agent_bullet_fire == True:  last_agent_bullet_x = agent_bullet_x; last_agent_bullet_y = agent_bullet_y  agent_bullet_x += agent_bullet_direction_x; agent_bullet_y += agent_bullet_direction_y if bullet_hit_detector("agent"): print "Bot Got Hit!" bot_hp -= bullet_damage reward = 100  agent_bullet_fire = False agent_bullet_direction_x = 0; agent_bullet_direction_y = 0 agent_bullet_x = agent_bullet_y = bullet_ob; last_agent_bullet_x = last_agent_bullet_y = bullet_ob if bot_hp <= 0: successful = True cont = False winner = "Agent"  elif agent_bullet_x + bullet_radius >= disp_x/2 + arena_x/2 or agent_bullet_x - bullet_radius <= disp_x/2 - arena_x/2 or \ agent_bullet_y + bullet_radius >= disp_y/2 + arena_y/2 or agent_bullet_y - bullet_radius <= disp_y/2 - arena_y/2: agent_bullet_fire = False agent_bullet_direction_x = 0; agent_bullet_direction_y = 0 agent_bullet_x = agent_bullet_y = bullet_ob; last_agent_bullet_x = last_agent_bullet_y = bullet_ob  return reward, cont, successful, winner #Parameters y = 0.75 e = 0.3 num_episodes = 10000 batch_size = 10 complexity = 10 with tf.Session() as sess: sess.run(initialize) success = 0 for i in range(1, num_episodes): rAll = 0; d = False; c = True; j = 0 param_init() samples = [] while c == True: j += 1  screen_blit()  current_state = np.array([[mapping(complexity, float(agent_x) / float(arena_x)), mapping(complexity, float(agent_y) / float(arena_y)), mapping(complexity, float(bot_x) / float(arena_x)), mapping(complexity, float(bot_y) / float(arena_y)), mapping(complexity, float(bot_bullet_x / float(arena_x))), mapping(complexity, float(bot_bullet_y / float(arena_y))), mapping(complexity, abs(float(agent_x - bot_x)) / float(arena_x)), mapping(complexity, abs(float(agent_y - bot_y)) / float(arena_y))]]) bot_take_action() if np.random.rand(1) < e or i <= 5: a = random.randint(0, 8) else: a, _ = sess.run([predict, Q],feed_dict={input_layer : current_state})    r, c, d, winner = action(a + 1, bot_action) next_state = np.array([[mapping(complexity, float(agent_x) / float(arena_x)), mapping(complexity, float(agent_y) / float(arena_y)), mapping(complexity, float(bot_x) / float(arena_x)), mapping(complexity, float(bot_y) / float(arena_y)), mapping(complexity, float(bot_bullet_x / float(arena_x))), mapping(complexity, float(bot_bullet_y / float(arena_y))), mapping(complexity, abs(float(agent_x - bot_x)) / float(arena_x)), mapping(complexity, abs(float(agent_y - bot_y)) / float(arena_y))]]) samples.append([current_state, a, r, next_state]) if len(samples) > 10: for count in xrange(batch_size): [batch_current_state, action_taken, reward, batch_next_state] = samples[random.randint(0, len(samples) - 1)] batch_allQ = sess.run(Q, feed_dict={input_layer : batch_current_state}) batch_Q1 = sess.run(Q, feed_dict = {input_layer : batch_next_state}) batch_maxQ1 = np.max(batch_Q1) batch_targetQ = batch_allQ batch_targetQ[0][a] = reward + y * batch_maxQ1 sess.run([updateModel], feed_dict={input_layer : batch_current_state, next_Q : batch_targetQ}) rAll += r if d == True: e = 1. / ((i / 50) + 10) success += 1 break display.update() rList.append(rAll) print winner  print "Successful episodes: %d out of %d. Success Rate = %d" % (success, num_episodes, float(success)/float(num_episodes)) plt.plot(rList) plt.show() 

from pygame import * from pygame.locals import * import sys from time import sleep import numpy as np import random import tensorflow as tf from pylab import savefig from tqdm import tqdm  #Screen Setup disp_x, disp_y = 1000, 800 arena_x, arena_y = 1000, 800 border = 4; border_2 = 1 #Color Setup white = (255, 255, 255); aqua= (0, 200, 200) red = (255, 0, 0); green = (0, 255, 0) blue = (0, 0, 255); black = (0, 0, 0) green_yellow = (173, 255, 47); energy_blue = (125, 249, 255)  #Initialize character positions init_character_a_state = [disp_x/2 - arena_x/2 + 50, disp_y/2 - arena_y/2 + 50] init_character_b_state = [disp_x/2 + arena_x/2 - 50, disp_y/2 + arena_y/2 - 50] #Setup character dimentions character_size = 50 character_move_speed = 25 #Initialize character stats character_init_health = 100 #initialize bullet stats beam_damage = 10 beam_width = 10 beam_ob = -100 #The Neural Network input_layer = tf.placeholder(shape=[1,7],dtype=tf.float32) weight_1 = tf.Variable(tf.random_uniform([7,9],0,0.1)) #weight_2 = tf.Variable(tf.random_uniform([6,9],0,0.1)) #The calculations, loss function and the update model Q = tf.matmul(input_layer, weight_1) predict = tf.argmax(Q, 1) next_Q = tf.placeholder(shape=[1,9],dtype=tf.float32) loss = tf.reduce_sum(tf.square(next_Q - Q)) trainer = tf.train.GradientDescentOptimizer(learning_rate=0.001) updateModel = trainer.minimize(loss) initialize = tf.global_variables_initializer() jList = [] rList = [] init() font.init() myfont = font.SysFont('Comic Sans MS', 15) myfont2 = font.SysFont('Comic Sans MS', 150) myfont3 = font.SysFont('Gothic', 30) disp = display.set_mode((disp_x, disp_y), 0, 32) #CHARACTER/BULLET PARAMETERS agent_x = agent_y = int() bot_x = bot_y = int() agent_hp = bot_hp = int() bot_beam_dir = int() agent_beam_fire = bot_beam_fire = bool() agent_beam_x = bot_beam_x = agent_beam_y = bot_beam_y = int() agent_beam_size_x = agent_beam_size_y = bot_beam_size_x = bot_beam_size_y = int() bot_current_action = agent_current_action = int() def param_init(): """Initializes parameters""" global agent_x, agent_y, bot_x, bot_y, agent_hp, bot_hp, agent_beam_fire, bot_beam_fire, agent_beam_x, bot_beam_x, agent_beam_y, bot_beam_y agent_x = list(init_character_a_state)[0]; agent_y = list(init_character_a_state)[1] bot_x = list(init_character_b_state)[0]; bot_y = list(init_character_b_state)[1] agent_hp = bot_hp = character_init_health  agent_beam_fire = bot_beam_fire = False agent_beam_x = bot_beam_x = agent_beam_y = bot_beam_y = beam_ob agent_beam_size_x = agent_beam_size_y = bot_beam_size_x = bot_beam_size_y = 0  def screen_blit(): global disp, disp_x, disp_y, arena_x, arena_y, border, border_2, character_size, agent_x, \ agent_y, bot_x, bot_y, character_init_health, agent_hp, bot_hp, red, blue, aqua, green, black, green_yellow, energy_blue, \ agent_beam_fire, bot_beam_fire, agent_beam_x, agent_beam_y, bot_beam_x, bot_beam_y, agent_beam_size_x, agent_beam_size_y, bot_beam_size_x, bot_beam_size_y, beam_width disp.fill(aqua) draw.rect(disp, black, (disp_x / 2 - arena_x / 2 - border, disp_y / 2 - arena_y / 2 - border, arena_x + border * 2, arena_y + border * 2)) draw.rect(disp, green, (disp_x / 2 - arena_x / 2, disp_y / 2 - arena_y / 2, arena_x, arena_y)) if bot_beam_fire == True: draw.rect(disp, green_yellow, (agent_beam_x, agent_beam_y, agent_beam_size_x, agent_beam_size_y))  bot_beam_fire = False  if agent_beam_fire == True:  draw.rect(disp, energy_blue, (bot_beam_x, bot_beam_y, bot_beam_size_x, bot_beam_size_y)) agent_beam_fire = False  draw.rect(disp, red, (agent_x, agent_y, character_size, character_size)) draw.rect(disp, blue, (bot_x, bot_y, character_size, character_size))   draw.rect(disp, red, (disp_x / 2 - 200, disp_y / 2 + arena_y / 2 + border + 1, float(agent_hp) / float(character_init_health) * 100, 14)) draw.rect(disp, blue, (disp_x / 2 + 200, disp_y / 2 + arena_y / 2 + border + 1, float(bot_hp) / float(character_init_health) * 100, 14))    def bot_take_action(): return random.randint(1, 9) def beam_hit_detector(player): global agent_x, agent_y, bot_x, bot_y, agent_beam_fire, bot_beam_fire, agent_beam_x, \ bot_beam_x, agent_beam_y, bot_beam_y, agent_beam_size_x, agent_beam_size_y, \  bot_beam_size_x, bot_beam_size_y, bot_current_action, agent_current_action, beam_width, character_size if player == "bot": if bot_current_action == 1: if disp_y/2 - arena_y/2 <= agent_y <= bot_y and (agent_x < bot_beam_x + beam_width < agent_x + character_size or agent_x < bot_beam_x < agent_x + character_size): return True else: return False elif bot_current_action == 2:  if bot_x <= agent_x <= disp_x/2 + arena_x/2 and (agent_y < bot_beam_y + beam_width < agent_y + character_size or agent_y < bot_beam_y < agent_y + character_size): return True else: return False elif bot_current_action == 3: if bot_y <= agent_y <= disp_y/2 + arena_y/2 and (agent_x < bot_beam_x + beam_width < agent_x + character_size or agent_x < bot_beam_x < agent_x + character_size): return True else: return False elif bot_current_action == 4: if disp_x/2 - arena_x/2 <= agent_x <= bot_x and (agent_y < bot_beam_y + beam_width < agent_y + character_size or agent_y < bot_beam_y < agent_y + character_size): return True else: return False else: if agent_current_action == 1: if disp_y/2 - arena_y/2 <= bot_y <= agent_y and (bot_x < agent_beam_x + beam_width < bot_x + character_size or bot_x < agent_beam_x < bot_x + character_size): return True   else: return False  elif agent_current_action == 2:  if agent_x <= bot_x <= disp_x/2 + arena_x/2 and (bot_y < agent_beam_y + beam_width < bot_y + character_size or bot_y < agent_beam_y < bot_y + character_size):  return True   else:  return False elif agent_current_action == 3:   if agent_y <= bot_y <= disp_y/2 + arena_y/2 and (bot_x < agent_beam_x + beam_width < bot_x + character_size or bot_x < agent_beam_x < bot_x + character_size):  return True   else: return False elif bot_current_action == 4:  if disp_x/2 - arena_x/2 <= bot_x <= agent_x and (bot_y < agent_beam_y + beam_width < bot_y + character_size or bot_y < agent_beam_y < bot_y + character_size): return True  else: return False def mapping(maximum, number): return number#int(number * maximum) def action(agent_action, bot_action): global agent_x, agent_y, bot_x, bot_y, agent_hp, bot_hp, agent_beam_fire, \ bot_beam_fire, agent_beam_x, bot_beam_x, agent_beam_y, bot_beam_y, agent_beam_size_x, \ agent_beam_size_y, bot_beam_size_x, bot_beam_size_y, beam_width, agent_current_action, bot_current_action, character_size  agent_current_action = agent_action; bot_current_action = bot_action reward = 0; cont = True; successful = False; winner = "" if 1 <= bot_action <= 4: bot_beam_fire = True if bot_action == 1: bot_beam_x = bot_x + character_size/2 - beam_width/2; bot_beam_y = disp_y/2 - arena_y/2 bot_beam_size_x = beam_width; bot_beam_size_y = bot_y - disp_y/2 + arena_y/2 elif bot_action == 2: bot_beam_x = bot_x + character_size; bot_beam_y = bot_y + character_size/2 - beam_width/2 bot_beam_size_x = disp_x/2 + arena_x/2 - bot_x - character_size; bot_beam_size_y = beam_width elif bot_action == 3: bot_beam_x = bot_x + character_size/2 - beam_width/2; bot_beam_y = bot_y + character_size bot_beam_size_x = beam_width; bot_beam_size_y = disp_y/2 + arena_y/2 - bot_y - character_size elif bot_action == 4: bot_beam_x = disp_x/2 - arena_x/2; bot_beam_y = bot_y + character_size/2 - beam_width/2  bot_beam_size_x = bot_x - disp_x/2 + arena_x/2; bot_beam_size_y = beam_width elif 5 <= bot_action <= 8: if bot_action == 5: bot_y -= character_move_speed if bot_y <= disp_y/2 - arena_y/2: bot_y = disp_y/2 - arena_y/2 elif agent_y <= bot_y <= agent_y + character_size:  bot_y = agent_y + character_size elif bot_action == 6: bot_x += character_move_speed if bot_x >= disp_x/2 + arena_x/2 - character_size: bot_x = disp_x/2 + arena_x/2 - character_size  elif agent_x <= bot_x + character_size <= agent_x + character_size: bot_x = agent_x - character_size elif bot_action == 7: bot_y += character_move_speed if bot_y + character_size >= disp_y/2 + arena_y/2: bot_y = disp_y/2 + arena_y/2 - character_size elif agent_y <= bot_y + character_size <= agent_y + character_size: bot_y = agent_y - character_size elif bot_action == 8: bot_x -= character_move_speed if bot_x <= disp_x/2 - arena_x/2: bot_x = disp_x/2 - arena_x/2 elif agent_x <= bot_x <= agent_x + character_size:  bot_x = agent_x + character_size if bot_beam_fire == True: if beam_hit_detector("bot"): #print "Agent Got Hit!" agent_hp -= beam_damage reward += -50 bot_beam_size_x = bot_beam_size_y = 0 bot_beam_x = bot_beam_y = beam_ob if agent_hp <= 0: cont = False winner = "Bot" if 1 <= agent_action <= 4: agent_beam_fire = True if agent_action == 1: if agent_y > disp_y/2 - arena_y/2: agent_beam_x = agent_x - beam_width/2; agent_beam_y = disp_y/2 - arena_y/2  agent_beam_size_x = beam_width; agent_beam_size_y = agent_y - disp_y/2 + arena_y/2  else: reward += -25 elif agent_action == 2: if agent_x + character_size < disp_x/2 + arena_x/2: agent_beam_x = agent_x + character_size; agent_beam_y = agent_y + character_size/2 - beam_width/2 agent_beam_size_x = disp_x/2 + arena_x/2 - agent_x - character_size; agent_beam_size_y = beam_width  else: reward += -25 elif agent_action == 3: if agent_y + character_size < disp_y/2 + arena_y/2: agent_beam_x = agent_x + character_size/2 - beam_width/2; agent_beam_y = agent_y + character_size agent_beam_size_x = beam_width; agent_beam_size_y = disp_y/2 + arena_y/2 - agent_y - character_size  else: reward += -25 elif agent_action == 4: if agent_x > disp_x/2 - arena_x/2: agent_beam_x = disp_x/2 - arena_x/2; agent_beam_y = agent_y + character_size/2 - beam_width/2 agent_beam_size_x = agent_x - disp_x/2 + arena_x/2; agent_beam_size_y = beam_width else: reward += -25  elif 5 <= agent_action <= 8: if agent_action == 5: agent_y -= character_move_speed if agent_y <= disp_y/2 - arena_y/2: agent_y = disp_y/2 - arena_y/2 reward += -5 elif bot_y <= agent_y <= bot_y + character_size and bot_x <= agent_x <= bot_x + character_size: agent_y = bot_y + character_size reward += -2 elif agent_action == 6: agent_x += character_move_speed if agent_x + character_size >= disp_x/2 + arena_x/2: agent_x = disp_x/2 + arena_x/2 - character_size reward += -5 elif bot_x <= agent_x + character_size <= bot_x + character_size and bot_y <= agent_y <= bot_y + character_size: agent_x = bot_x - character_size reward += -2 elif agent_action == 7: agent_y += character_move_speed if agent_y + character_size >= disp_y/2 + arena_y/2: agent_y = disp_y/2 + arena_y/2 - character_size reward += -5 elif bot_y <= agent_y + character_size <= bot_y + character_size and bot_x <= agent_x <= bot_x + character_size: agent_y = bot_y - character_size reward += -2 elif agent_action == 8: agent_x -= character_move_speed if agent_x <= disp_x/2 - arena_x/2: agent_x = disp_x/2 - arena_x/2 reward += -5 elif bot_x <= agent_x <= bot_x + character_size and bot_y <= agent_y <= bot_y + character_size:  agent_x = bot_x + character_size    reward += -2  if agent_beam_fire == True: if beam_hit_detector("agent"): #print "Bot Got Hit!" bot_hp -= beam_damage reward += 50 agent_beam_size_x = agent_beam_size_y = 0 agent_beam_x = agent_beam_y = beam_ob if bot_hp <= 0: successful = True cont = False winner = "Agent" return reward, cont, successful, winner def bot_beam_dir_detector(): global bot_current_action if bot_current_action == 1: bot_beam_dir = 2 elif bot_current_action == 2: bot_beam_dir = 4 elif bot_current_action == 3: bot_beam_dir = 3 elif bot_current_action == 4: bot_beam_dir = 1 else: bot_beam_dir = 0 return bot_beam_dir  #Parameters y = 0.75 e = 0.3 num_episodes = 10000 batch_size = 10 complexity = 100 with tf.Session() as sess: sess.run(initialize) success = 0 for i in tqdm(range(1, num_episodes)): #print "Episode #", i rAll = 0; d = False; c = True; j = 0 param_init() samples = [] while c == True: j += 1 current_state = np.array([[mapping(complexity, float(agent_x) / float(arena_x)), mapping(complexity, float(agent_y) / float(arena_y)), mapping(complexity, float(bot_x) / float(arena_x)), mapping(complexity, float(bot_y) / float(arena_y)), #mapping(complexity, float(agent_hp) / float(character_init_health)), #mapping(complexity, float(bot_hp) / float(character_init_health)), mapping(complexity, float(agent_x - bot_x) / float(arena_x)), mapping(complexity, float(agent_y - bot_y) / float(arena_y)), bot_beam_dir ]])   b = bot_take_action() if np.random.rand(1) < e or i <= 5: a = random.randint(0, 8) else: a, _ = sess.run([predict, Q],feed_dict={input_layer : current_state}) r, c, d, winner = action(a + 1, b) bot_beam_dir = bot_beam_dir_detector() next_state = np.array([[mapping(complexity, float(agent_x) / float(arena_x)), mapping(complexity, float(agent_y) / float(arena_y)), mapping(complexity, float(bot_x) / float(arena_x)), mapping(complexity, float(bot_y) / float(arena_y)), #mapping(complexity, float(agent_hp) / float(character_init_health)), #mapping(complexity, float(bot_hp) / float(character_init_health)), mapping(complexity, float(agent_x - bot_x) / float(arena_x)), mapping(complexity, float(agent_y - bot_y) / float(arena_y)), bot_beam_dir ]]) samples.append([current_state, a, r, next_state]) if len(samples) > 10: for count in xrange(batch_size): [batch_current_state, action_taken, reward, batch_next_state] = samples[random.randint(0, len(samples) - 1)] batch_allQ = sess.run(Q, feed_dict={input_layer : batch_current_state}) batch_Q1 = sess.run(Q, feed_dict = {input_layer : batch_next_state}) batch_maxQ1 = np.max(batch_Q1) batch_targetQ = batch_allQ batch_targetQ[0][a] = reward + y * batch_maxQ1 sess.run([updateModel], feed_dict={input_layer : batch_current_state, next_Q : batch_targetQ}) rAll += r  screen_blit()  if d == True: e = 1. / ((i / 50) + 10) success += 1 break  #print agent_hp, bot_hp  display.update()  jList.append(j)  rList.append(rAll) print winner 

Update #3 on August 22, 2017:

I've noticed that if the agent hits the bot with a bullet on a turn and the action the bot taken on that turn was not "fire a bullet", then the wrong actions would be given credit. Thus, I've turned the bullets into beams so the bot/agent takes damage on the turn the beam's fired.

Update:

Update #1 on August 14th, 2017:

Update #2 on August 18, 2017:

Based on the advice of @NeilSlater, I've implemented experience replay into my model. The algorithm has improved, but I'm going to look for more better improvement options that offer convergence.

from pygame import * from pygame.locals import * import sys from time import sleep import numpy as np import random from time import sleep import tensorflow as tf #Screen Setup disp_x, disp_y = 1500, 1000 arena_x, arena_y = 800, 800 border = 4; border_2 = 1 #Color Setup white = (255, 255, 255); aqua= (0, 200, 200); red = (255, 0, 0); green = (0, 255, 0); blue = (0, 0, 255); black = (0, 0, 0) #Initialize character positions init_character_a_state = [disp_x/2 - arena_x/2 + 50, disp_y/2 - arena_y/2 + 50] init_character_b_state = [disp_x/2 + arena_x/2 - 50, disp_y/2 + arena_y/2 - 50] #Setup character dimentions character_radius = 30 character_move_speed = 20 #Initialize character stats character_init_health = 100 #initialize bullet stats bullet_speed = 30 bullet_damage = 10 bullet_radius = 7 bullet_a_pos = list(init_character_a_state); bullet_b_pos = list(init_character_b_state) bullet_a_fire = False; bullet_b_fire = False #The Neural Network input_layer = tf.placeholder(shape=[1,8],dtype=tf.float32) weight_1 = tf.Variable(tf.random_uniform([8,9],0,0.1)) #The calculations, loss function and the update model Q = tf.matmul(input_layer, weight_1) predict = tf.argmax(Q, 1) next_Q = tf.placeholder(shape=[1,9],dtype=tf.float32) loss = tf.reduce_sum(tf.square(next_Q - Q)) trainer = tf.train.GradientDescentOptimizer(learning_rate=0.0001) updateModel = trainer.minimize(loss) initialize = tf.global_variables_initializer() #Parameters y = 0.75 e = 0.3 num_episodes = 10000 batch_size = 10  jList = [] rList = [] init() font.init() myfont = font.SysFont('Comic Sans MS', 15) myfont2 = font.SysFont('Comic Sans MS', 150) myfont3 = font.SysFont('Gothic', 30) disp = display.set_mode((disp_x, disp_y), 0, 32) def mapping(maximum, number): return int(abs(number * maximum) / (maximum/10))  def clip(value): if value > 250.0: value = 250.0 elif value < -250.0: value = -250.0 return value  #Environment (Training) Parameters: #agent_bullet_fire = bot_bullet_fire = False #agent_bullet_dir = [0, 0]; bot_bullet_dir = [0, 0] #The environment: def action(agent_x, agent_y, bot_x, bot_y, agent_action, bot_action, agent_hp, bot_hp, agent_bullet, bot_bullet, agent_bullet_fire, bot_bullet_fire, agent_bullet_dir, bot_bullet_dir):  #Bullet Management reward = 0  cont = True  successful = False  winner = "" if 1 <= bot_action <= 4 and bot_bullet_fire == False:  #If bullet's fired by bot:  bot_bullet_fire = True if bot_action == 1: bot_bullet_dir = [0, -bullet_speed] elif bot_action == 2: bot_bullet_dir = [bullet_speed, 0] elif bot_action == 3: bot_bullet_dir = [0, bullet_speed] elif bot_action == 4: bot_bullet_dir = [-bullet_speed, 0] elif bot_bullet_fire == True: bot_bullet[0] = bot_bullet[0] + bot_bullet_dir[0]; bot_bullet[1] = bot_bullet[1] + bot_bullet_dir[1] if bot_bullet[0] > disp_x/2 + arena_x/2 + bullet_radius or bot_bullet[0] < disp_x/2 - arena_x/2 - bullet_radius or bot_bullet[1] > disp_y/2 + arena_y/2 + bullet_radius or bot_bullet[1] < disp_y/2 - arena_y/2 - bullet_radius:  bot_bullet_fire = False bot_bullet = [bot_x, bot_y] if agent_x - character_radius - border <= bot_bullet[0] <= agent_x + character_radius + border and agent_y - character_radius - border < bot_bullet[1] < agent_y + character_radius + border: agent_hp -= bullet_damage  reward = -50 if agent_hp <= 0:  cont = False   winner = "Bot" if 5 <= bot_action <= 8: bot_bullet = [bot_x, bot_y] if bot_action == 5: bot_y -= character_move_speed if bot_y <= disp_y/2 - arena_y/2 + character_radius + 1: bot_y = disp_y/2 - arena_y/2 + character_radius + 1 elif bot_action == 6: bot_x += character_move_speed if bot_x >= disp_x/2 + arena_x/2 - character_radius - 1: bot_x = disp_x/2 + arena_x/2 - character_radius - 1 elif bot_action == 7: bot_y += character_move_speed if bot_y >= disp_y/2 + arena_y/2 - character_radius - 1: bot_y = disp_y/2 + arena_y/2 - character_radius - 1 elif bot_action == 8: bot_x -= character_move_speed if bot_x <= disp_x/2 - arena_x/2 + character_radius + 1: bot_x = disp_x/2 - arena_x/2 + character_radius + 1    if 1 <= agent_action <= 4 and agent_bullet_fire == False: agent_bullet_fire = True  if agent_action == 1:  if agent_y - character_radius > disp_y/2 - arena_y + border:   agent_bullet_dir = [0, -bullet_speed] else:  reward = -25 elif agent_action == 2: if agent_x + character_radius < disp_x/2 + arena_x - border:   agent_bullet_dir = [bullet_speed, 0]  else:   reward = -25  elif agent_action == 3: if agent_y + character_radius < disp_y/2 + arena_y - border: agent_bullet_dir = [0, bullet_speed]  else: reward = -25 elif agent_action == 4:  if agent_x - character_radius > disp_x/2 - arena_x + border:   agent_bullet_dir = [-bullet_speed, 0]  else:   reward = -25   elif 1 <= agent_action <= 4 and agent_bullet_fire == True:  reward = -20   elif agent_bullet_fire == True:  agent_bullet[0] = agent_bullet[0] + agent_bullet_dir[0]; agent_bullet[1] = agent_bullet[1] + agent_bullet_dir[1]   if agent_bullet[0] > disp_x/2 + arena_x/2 + bullet_radius or agent_bullet[0] < disp_x/2 - arena_x/2 - bullet_radius or agent_bullet[1] > disp_y/2 + arena_y/2 + bullet_radius or agent_bullet[1] < disp_y/2 - arena_y/2 - bullet_radius: agent_bullet_fire = False agent_bullet = [agent_x, agent_y] agent_bullet_dir = [0, 0] if bot_x - character_radius <= agent_bullet[0] <= bot_x + character_radius and bot_y - character_radius < agent_bullet[1] < bot_y + character_radius: bot_hp -= bullet_damage reward = 100 agent_bullet_fire = False agent_bullet_dir = [0, 0] if bot_hp <= 0: successful = True cont = False winner = "Agent"  if 5 <= agent_action <= 8: agent_bullet = [agent_x, agent_y] if agent_action == 5: agent_y -= character_move_speed if agent_y <= disp_y/2 - arena_y/2 + character_radius + 1: agent_y = disp_y/2 - arena_y/2 + character_radius + 1 reward = -5 else: reward = 5 elif agent_action == 6: agent_x += character_move_speed if agent_x >= disp_x/2 + arena_x/2 - character_radius - 1: agent_x = disp_x/2 + arena_x/2 - character_radius - 1 reward = -5 else: reward = 5 elif agent_action == 7: agent_y += character_move_speed if agent_y >= disp_y/2 + arena_y/2 - character_radius - 1: agent_y = disp_y/2 + arena_y/2 - character_radius - 1 reward = -5 else: reward = 5 elif agent_action == 8: agent_x -= character_move_speed if agent_x <= disp_x/2 - arena_x/2 + character_radius + 1: agent_x = disp_x/2 - arena_x/2 + character_radius + 1 reward = -5 else: reward = 5  return reward, cont, successful, agent_x, agent_y, bot_x, bot_y, agent_hp, bot_hp, agent_bullet, bot_bullet, winner, agent_bullet_fire, bot_bullet_fire, agent_bullet_dir, bot_bullet_dir  with tf.Session() as sess: sess.run(initialize) success = 0 for i in range(num_episodes): rAll = 0  s = 0  d = False  agent_x = int(list(init_character_a_state)[0]); agent_y = int(list(init_character_a_state)[1]) bot_x = int(list(init_character_b_state)[0]); bot_y = int(list(init_character_b_state)[1])  agent_hp = bot_hp = int(character_init_health) bot_bullet = list(init_character_b_state); agent_bullet = list(init_character_a_state) agent_bullet_fire = bot_bullet_fire = False agent_bullet_dir = [0, 0]; bot_bullet_dir = [0, 0]  j = 0   c = True samples = [] while c == True:  disp.fill(aqua) draw.rect(disp, black, (disp_x/2 - arena_x/2 - border, disp_y/2 - arena_y/2 - border, arena_x + border * 2, arena_y + border * 2)) draw.rect(disp, green, (disp_x/2 - arena_x/2, disp_y/2 - arena_y/2, arena_x, arena_y)) draw.circle(disp, black, agent_bullet, bullet_radius) draw.circle(disp, black, bot_bullet, bullet_radius) draw.circle(disp, black, (agent_x, agent_y), character_radius + border_2) draw.circle(disp, red, (agent_x, agent_y), character_radius) draw.circle(disp, black, (bot_x, bot_y), character_radius + border_2) draw.circle(disp, blue, (bot_x, bot_y), character_radius) draw.rect(disp, red, (disp_x / 2 - 200, disp_y / 2 + arena_y / 2 + border + 1, float(agent_hp)/float(character_init_health) * 100, 14)) draw.rect(disp, blue, (disp_x / 2 + 200, disp_y / 2 + arena_y / 2 + border + 1, float(bot_hp)/float(character_init_health) * 100, 14))  j += 1 """ ---CURRENT STATE--- Evenything will be on a scale of 0 to "complexity". (0 = 0, "complexity" = max) """ complexity = 100 current_state = np.array([[mapping(complexity, float(agent_x) / float(arena_x)), mapping(complexity, float( agent_y) / float(arena_y)), mapping(complexity, float( bot_x) / float(arena_x)), mapping(complexity, float( bot_y) / float(arena_y)), mapping(complexity, float( bot_bullet[0] / float(arena_x))), mapping(complexity, float( bot_bullet[1] / float(arena_y))), mapping(complexity, abs( float(agent_x - bot_x)) / float(arena_x)), mapping(complexity, abs(float(agent_y - bot_y)) / float(arena_y))]])   #current_state is the array of parameters for feeding the neural network  #print current_state a,allQ = sess.run([predict, Q],feed_dict={input_layer : current_state}) #bot move #a[0] = moves[0]   #moves = moves[1:] #1~4 are shooting a bullet. 5~8 are movement. 9 is doing nothing. if agent_x - character_radius <= bot_x <= agent_x + character_radius:  if agent_y <= bot_y: if random.randint(0, 100) > 20: bot_action = 1 else: if random.randint(0, 100) > 20:  bot_action = 3 elif agent_y - character_radius <= bot_y <= agent_y + character_radius:  if agent_x <= bot_x: if random.randint(0, 100) > 20:   bot_action = 4 else:  if random.randint(0, 100) > 20: bot_action = 2 else: if random.randint(0, 100) > 20: #Find opponent, caluculate x and y distance and go the shortest way  x_dist = abs(bot_x - agent_x); y_dist = abs(bot_y - agent_y) if x_dist >= y_dist: if bot_x - agent_x <= 0: bot_action = 6 else: bot_action = 8 else: if bot_y - agent_y <= 0: bot_action = 7 else: bot_action = 5  else:  bot_action = random.randint(1, 9) if np.random.rand(1) < e: a[0] = random.randint(0,8) #Action: Takes positions and actions. r, c, d, new_agent_x, new_agent_y, new_bot_x, new_bot_y, new_agent_hp, new_bot_hp, new_agent_bullet, new_bot_bullet, winner, agent_bullet_fire, bot_bullet_fire, agent_bullet_dir, bot_bullet_dir = action(agent_x, agent_y, bot_x, bot_y, int(a[0]+1), bot_action, agent_hp, bot_hp, agent_bullet, bot_bullet, agent_bullet_fire, bot_bullet_fire, agent_bullet_dir, bot_bullet_dir) next_state = np.array([[mapping(complexity, float(new_agent_x) / float(arena_x)), mapping(complexity, float( new_agent_y) / float(arena_y)), mapping(complexity, float( new_bot_x) / float(arena_x)), mapping(complexity, float( new_bot_y) / float(arena_y)), mapping(complexity, float( new_bot_bullet[0] / float(arena_x))), mapping(complexity, float( new_bot_bullet[1] / float(arena_y))), mapping(complexity, float( new_agent_x - new_bot_x) / float(disp_x)), mapping(complexity, float(new_agent_y - new_bot_y) / float(disp_y))]]) #Q1 = sess.run(Q, feed_dict = {input_layer : next_state}) #maxQ1 = np.max(Q1) #targetQ = allQ #targetQ[0,a[0]] = r + y * maxQ1 #for index, buf in enumerate(targetQ[0]): # targetQ[0][index] = clip(targetQ[0][index]) samples.append([current_state, a, r, next_state])  #print samples #print len(samples) allQ = sess.run(Q, feed_dict={input_layer : current_state}) print allQ  if len(samples) > 10: for count in xrange(batch_size): [batch_current_state, action_taken, reward, batch_next_state] = samples[random.randint(0, len(samples) - 1)] batch_allQ = sess.run(Q, feed_dict={input_layer : batch_current_state}) batch_Q1 = sess.run(Q, feed_dict = {input_layer : batch_next_state}) batch_maxQ1 = np.max(batch_Q1) batch_targetQ = batch_allQ #print batch_targetQ batch_targetQ[0,a[0]] = reward + y * batch_maxQ1 sess.run([updateModel], feed_dict={input_layer : batch_current_state, next_Q : batch_targetQ})  #print a[0] + 1, r, targetQ[0]  rAll += r  bot_x = new_bot_x; bot_y = new_bot_y; agent_x = new_agent_x; agent_y = new_agent_y; agent_hp = new_agent_hp; bot_hp = new_bot_hp; agent_bullet = new_agent_bullet; bot_bullet = new_bot_bullet  if d == True: e = 1./((i/50) + 10) success += 1 break display.update()  #sleep(1) jList.append(j)  rList.append(rAll) print winner plt.plot(rList) plt.show() 

from pygame import * from pygame.locals import * import sys from time import sleep import numpy as np import random from time import sleep import tensorflow as tf #Screen Setup disp_x, disp_y = 1500, 1000 arena_x, arena_y = 800, 800 border = 4; border_2 = 1 #Color Setup white = (255, 255, 255); aqua= (0, 200, 200)  red = (255, 0, 0); green = (0, 255, 0)  blue = (0, 0, 255); black = (0, 0, 0) #Initialize character positions init_character_a_state = [disp_x/2 - arena_x/2 + 50, disp_y/2 - arena_y/2 + 50] init_character_b_state = [disp_x/2 + arena_x/2 - 50, disp_y/2 + arena_y/2 - 50] #Setup character dimentions character_radius = 30 character_move_speed = 20 #Initialize character stats character_init_health = 100 #initialize bullet stats bullet_speed = 50 bullet_damage = 10 bullet_radius = 7 bullet_a_pos = list(init_character_a_state); bullet_b_pos = list(init_character_b_state) bullet_a_fire = False; bullet_b_fire = False #The Neural Network input_layer = tf.placeholder(shape=[1,8],dtype=tf.float32) weight_1 = tf.Variable(tf.random_uniform([8,9],0,0.1)) #The calculations, loss function and the update model Q = tf.matmul(input_layer, weight_1) predict = tf.argmax(Q, 1) next_Q = tf.placeholder(shape=[1,9],dtype=tf.float32) loss = tf.reduce_sum(tf.square(next_Q - Q)) trainer = tf.train.GradientDescentOptimizer(learning_rate=0.0001) updateModel = trainer.minimize(loss) initialize = tf.global_variables_initializer() jList = [] rList = [] init() font.init() myfont = font.SysFont('Comic Sans MS', 15) myfont2 = font.SysFont('Comic Sans MS', 150) myfont3 = font.SysFont('Gothic', 30) disp = display.set_mode((disp_x, disp_y), 0, 32)  bullet_ob = -100 #CHARACTER/BULLET PARAMETERS bot_bullet_x = bot_bullet_y = bullet_ob agent_bullet_x = agent_bullet_y = bullet_ob last_bot_bullet_x = last_bot_bullet_y = bullet_ob last_agent_bullet_x = last_agent_bullet_y = bullet_ob agent_bullet_fire = bot_bullet_fire = bool() agent_bullet_direction_x = agent_bullet_direction_y = int() bot_bullet_direction_x = bot_bullet_direction_y = int() agent_x = agent_y = int() bot_x = bot_y = int() agent_hp = bot_hp = int() def param_init(): """Initializes parameters""" global bot_bullet_x, bot_bullet_y, agent_bullet_x, agent_bullet_y, agent_bullet_fire, \ bot_bullet_fire, agent_bullet_direction_x, agent_bullet_direction_y, bot_bullet_direction_x, \ bot_bullet_direction_y, agent_x, agent_y, bot_x, bot_y, agent_hp, bot_hp agent_bullet_x = agent_bullet_y = bullet_ob bot_bullet_x = bot_bullet_y = bullet_ob last_agent_bullet_x = last_agent_bullet_y = bullet_ob last_bot_bullet_x = last_bot_bullet_y = bullet_ob agent_bullet_fire = bot_bullet_fire = False agent_bullet_direction_x = 0; agent_bullet_direction_y = 0 bot_bullet_direction_x = 0; bot_bullet_direction_y = 0 agent_x = list(init_character_a_state)[0]; agent_y = list(init_character_a_state)[1] bot_x = list(init_character_b_state)[0]; bot_y = list(init_character_b_state)[1] agent_hp = bot_hp = character_init_health def screen_blit(): global disp, disp_x, disp_y, arena_x, arena_y, border, border_2, agent_bullet_x, \ agent_bullet_y, bullet_radius, bot_bullet_x, bot_bullet_y, character_radius, agent_x, \ agent_y, bot_x, bot_y, character_init_health, agent_hp, bot_hp, red, blue, aqua, green, black disp.fill(aqua) draw.rect(disp, black, (disp_x / 2 - arena_x / 2 - border, disp_y / 2 - arena_y / 2 - border, arena_x + border * 2, arena_y + border * 2)) draw.rect(disp, green, (disp_x / 2 - arena_x / 2, disp_y / 2 - arena_y / 2, arena_x, arena_y)) draw.circle(disp, black, [agent_bullet_x, agent_bullet_y], bullet_radius) draw.circle(disp, black, [bot_bullet_x, bot_bullet_y], bullet_radius) draw.circle(disp, black, (agent_x, agent_y), character_radius + border_2) draw.circle(disp, red, (agent_x, agent_y), character_radius) draw.circle(disp, black, (bot_x, bot_y), character_radius + border_2) draw.circle(disp, blue, (bot_x, bot_y), character_radius) draw.rect(disp, red, (disp_x / 2 - 200, disp_y / 2 + arena_y / 2 + border + 1, float(agent_hp) / float(character_init_health) * 100, 14)) draw.rect(disp, blue, (disp_x / 2 + 200, disp_y / 2 + arena_y / 2 + border + 1, float(bot_hp) / float(character_init_health) * 100, 14)) def bot_take_action(): global agent_x, agent_y, bot_x, bot_y, character_radius, bot_action, border if agent_x - character_radius - border <= bot_x <= agent_x + character_radius + border: if random.randint(0, 100) > 5: if agent_y <= bot_y: bot_action = 1 else: bot_action = 3 else: bot_action = 9 elif agent_y - character_radius <= bot_y <= agent_y + character_radius: if random.randint(0, 100) > 5: if agent_x <= bot_x: bot_action = 4 else: bot_action = 2 else: bot_action = 9 else: if random.randint(0, 100) > 5: x_dist = abs(bot_x - agent_x); y_dist = abs(bot_y - agent_y) if x_dist >= y_dist: if bot_x - agent_x <= 0: bot_action = 6 else: bot_action = 8 else: if bot_y - agent_y <= 0: bot_action = 7 else: bot_action = 5 else: bot_action = random.randint(1, 9) def bullet_hit_detector(player): global bot_bullet_x, bot_bullet_y, last_bot_bullet_x, last_bot_bullet_y, agent_x, agent_y, last_agent_bullet_x, last_agent_bullet_y, character_radius, border, bullet_radius if player == "bot": if bot_bullet_x == last_bot_bullet_x: if agent_x - character_radius - border < bot_bullet_x + bullet_radius < agent_x + character_radius + border or \ agent_x - character_radius - border < bot_bullet_x - bullet_radius < agent_x + character_radius + border: #If the current state of the bullet is touching/inside the agent: if agent_y - character_radius - border < bot_bullet_y + bullet_radius < agent_y + character_radius or \ agent_y - character_radius < bot_bullet_y - bullet_radius < agent_y + character_radius + border: return True #If the bullet "passed through" the character from the last turn: elif (last_bot_bullet_y - bullet_radius > agent_y + character_radius + border and agent_y - character_radius - border > bot_bullet_y + bullet_radius) \ or (bot_bullet_y - bullet_radius > agent_y + character_radius + border and agent_y - character_radius - border > last_bot_bullet_y + bullet_radius): return True else: return False elif bot_bullet_y == last_bot_bullet_y: if agent_y - character_radius - border < bot_bullet_y - bullet_radius < agent_y + character_radius + border or \ agent_y - character_radius - border < bot_bullet_y + bullet_radius < agent_y + character_radius + border: #If the current state of the bullet is touching/inside the agent: if agent_x - character_radius - border < bot_bullet_x + bullet_radius < agent_x + character_radius or \ agent_x - character_radius < bot_bullet_x - bullet_radius < agent_x + character_radius + border: return True #If the bullet "passed through" the character from the last turn: elif (last_bot_bullet_x - bullet_radius > agent_x + character_radius + border and agent_x - character_radius - border > bot_bullet_x + bullet_radius) \ or (bot_bullet_x - bullet_radius > agent_x + character_radius + border and agent_x - character_radius - border > last_bot_bullet_x + bullet_radius): return True else: return False else: if agent_bullet_x == last_agent_bullet_x: if bot_x - character_radius - border < agent_bullet_x + bullet_radius < bot_x + character_radius + border or \ bot_x - character_radius - border < agent_bullet_x - bullet_radius < bot_x + character_radius + border: #If the current state of the bullet is touching/inside the agent: if bot_y - character_radius - border < agent_bullet_y + bullet_radius < bot_y + character_radius or \ bot_y - character_radius < agent_bullet_y - bullet_radius < bot_y + character_radius + border: return True #If the bullet "passed through" the character from the last turn: elif (last_agent_bullet_y - bullet_radius > bot_y + character_radius + border and bot_y - character_radius - border > agent_bullet_y + bullet_radius) \ or (agent_bullet_y - bullet_radius > bot_y + character_radius + border and bot_y - character_radius - border > last_agent_bullet_y + bullet_radius): return True else: return False elif agent_bullet_y == last_agent_bullet_y: if bot_y - character_radius - border < agent_bullet_y - bullet_radius < bot_y + character_radius + border or \ bot_y - character_radius - border < agent_bullet_y + bullet_radius < bot_y + character_radius + border: #If the current state of the bullet is touching/inside the agent: if bot_x - character_radius - border < agent_bullet_x + bullet_radius < bot_x + character_radius or \ bot_x - character_radius < agent_bullet_x - bullet_radius < bot_x + character_radius + border: return True #If the bullet "passed through" the character from the last turn: elif (last_agent_bullet_x - bullet_radius > bot_x + character_radius + border and bot_x - character_radius - border > agent_bullet_x + bullet_radius) \ or (agent_bullet_x - bullet_radius > bot_x + character_radius + border and bot_x - character_radius - border > last_agent_bullet_x + bullet_radius): return True else: return False  def mapping(maximum, number): return int(abs(number * maximum) / (maximum/10)) def action(agent_action, bot_action): global bot_bullet_x, bot_bullet_y, agent_bullet_x, agent_bullet_y, agent_bullet_fire, \ bot_bullet_fire, agent_bullet_direction_x, agent_bullet_direction_y, bot_bullet_direction_x, \   bot_bullet_direction_y, agent_x, agent_y, bot_x, bot_y, agent_hp, bot_hp, last_agent_bullet_x, last_agent_bullet_y, last_bot_bullet_x, last_bot_bullet_y reward = 0; cont = True; successful = False; winner = "" if 1 <= bot_action <= 4 and bot_bullet_fire == False: bot_bullet_fire = True if bot_action == 1: bot_bullet_direction_x = 0; bot_bullet_direction_y = -bullet_speed elif bot_action == 2: bot_bullet_direction_x = bullet_speed; bot_bullet_direction_y = 0 elif bot_action == 3: bot_bullet_direction_x = 0; bot_bullet_direction_y = bullet_speed elif bot_action == 4: bot_bullet_direction_x = -bullet_speed; bot_bullet_direction_y = 0 bot_bullet_x = bot_x + bot_bullet_direction_x; bot_bullet_y = bot_y + bot_bullet_direction_y elif 5 <= bot_action <= 8: if bot_action == 5: bot_y -= character_move_speed if bot_y <= disp_y/2 - arena_y/2 + character_radius + 1: bot_y = disp_y/2 - arena_y/2 + character_radius + 1 elif bot_action == 6: bot_x += character_move_speed if bot_x >= disp_x/2 + arena_x/2 - character_radius - 1: bot_x = disp_x/2 + arena_x/2 - character_radius - 1 elif bot_action == 7: bot_y += character_move_speed if bot_y >= disp_y/2 + arena_y/2 - character_radius - 1: bot_y = disp_y/2 + arena_y/2 - character_radius - 1 elif bot_action == 8: bot_x -= character_move_speed if bot_x <= disp_x/2 - arena_x/2 + character_radius + 1: bot_x = disp_x/2 - arena_x/2 + character_radius + 1 if bot_bullet_fire == True: last_bot_bullet_x = bot_bullet_x; last_bot_bullet_y = bot_bullet_y  bot_bullet_x += bot_bullet_direction_x; bot_bullet_y += bot_bullet_direction_y    if bullet_hit_detector("bot"): print "Agent Got Hit!" agent_hp -= bullet_damage  reward = -50 bot_bullet_fire = False bot_bullet_direction_x = 0; bot_bullet_direction_y = 0 bot_bullet_x = bot_bullet_y = bullet_ob; last_bot_bullet_x = last_bot_bullet_y = bullet_ob if agent_hp <= 0: cont = False winner = "Bot" elif bot_bullet_x + bullet_radius >= disp_x/2 + arena_x/2 or bot_bullet_x - bullet_radius <= disp_x/2 - arena_x/2 or \ bot_bullet_y + bullet_radius >= disp_y/2 + arena_y/2 or bot_bullet_y - bullet_radius <= disp_y/2 - arena_y/2: bot_bullet_fire = False bot_bullet_direction_x = 0; bot_bullet_direction_y = 0 bot_bullet_x = bot_bullet_y = bullet_ob; last_bot_bullet_x = last_bot_bullet_y = bullet_ob if 1 <= agent_action <= 4: if agent_bullet_fire == False: agent_bullet_fire = True  if agent_action == 1: if agent_y - character_radius - border > disp_y/2 - arena_y/2: agent_bullet_direction_x = 0; agent_bullet_direction_y = -bullet_speed reward = 10 else: reward = -25  agent_bullet_x = agent_bullet_y = bullet_ob agent_bullet_fire = False elif agent_action == 2: if agent_x + character_radius + border < disp_x/2 + arena_x/2: agent_bullet_direction_x = bullet_speed; agent_bullet_direction_y = 0 reward = 10 else: reward = -25    agent_bullet_x = agent_bullet_y = bullet_ob agent_bullet_fire = False elif agent_action == 3: if agent_y + character_radius + border < disp_y/2 + arena_y/2:   agent_bullet_direction_x = 0; agent_bullet_direction_y = bullet_speed reward = 10 else:   reward = -25  agent_bullet_x = agent_bullet_y = bullet_ob   agent_bullet_fire = False   elif agent_action == 4:  if agent_x - character_radius - border > disp_x/2 - arena_x/2:   agent_bullet_direction_x = -bullet_speed; agent_bullet_direction_y = 0   reward = 10  else:  reward = -25   agent_bullet_x = agent_bullet_y = bullet_ob  agent_bullet_fire = False if agent_bullet_fire == True:   agent_bullet_x = agent_x + agent_bullet_direction_x; agent_bullet_y = agent_y + agent_bullet_direction_y   last_agent_bullet_x = agent_bullet_x; last_agent_bullet_y = agent_bullet_y   else:  reward = -20   elif 5 <= agent_action <= 8: if agent_action == 5: agent_y -= character_move_speed if agent_y - character_radius - border <= disp_y/2 - arena_y/2: agent_y = disp_y/2 - arena_y/2 + character_radius + border reward = -5 else: reward = 5 elif agent_action == 6: agent_x += character_move_speed if agent_x + character_radius + border >= disp_x/2 + arena_x/2: agent_x = disp_x/2 + arena_x/2 - character_radius - border reward = -5 else: reward = 5 elif agent_action == 7: agent_y += character_move_speed if agent_y + character_radius + border >= disp_y/2 + arena_y/2: agent_y = disp_y/2 + arena_y/2 - character_radius - border reward = -5 else: reward = 5 elif agent_action == 8: agent_x -= character_move_speed if agent_x - character_radius - border <= disp_x/2 - arena_x/2: agent_x = disp_x/2 - arena_x/2 + character_radius + border reward = -5 else: reward = 5  if agent_bullet_fire == True: last_agent_bullet_x = agent_bullet_x; last_agent_bullet_y = agent_bullet_y agent_bullet_x += agent_bullet_direction_x; agent_bullet_y += agent_bullet_direction_y if bullet_hit_detector("agent"): print "Bot Got Hit!" bot_hp -= bullet_damage reward = 100 agent_bullet_fire = False agent_bullet_direction_x = 0; agent_bullet_direction_y = 0 agent_bullet_x = agent_bullet_y = bullet_ob; last_agent_bullet_x = last_agent_bullet_y = bullet_ob if bot_hp <= 0: successful = True cont = False winner = "Agent" elif agent_bullet_x + bullet_radius >= disp_x/2 + arena_x/2 or agent_bullet_x - bullet_radius <= disp_x/2 - arena_x/2 or \ agent_bullet_y + bullet_radius >= disp_y/2 + arena_y/2 or agent_bullet_y - bullet_radius <= disp_y/2 - arena_y/2: agent_bullet_fire = False agent_bullet_direction_x = 0; agent_bullet_direction_y = 0 agent_bullet_x = agent_bullet_y = bullet_ob; last_agent_bullet_x = last_agent_bullet_y = bullet_ob return reward, cont, successful, winner  #Parameters y = 0.75 e = 0.3 num_episodes = 10000 batch_size = 10 complexity = 10 with tf.Session() as sess: sess.run(initialize) success = 0 for i in range(1, num_episodes): rAll = 0; d = False; c = True; j = 0 param_init() samples = [] while c == True: j += 1 screen_blit() current_state = np.array([[mapping(complexity, float(agent_x) / float(arena_x)),   mapping(complexity, float(agent_y) / float(arena_y)),   mapping(complexity, float(bot_x) / float(arena_x)),   mapping(complexity, float(bot_y) / float(arena_y)),   mapping(complexity, float(bot_bullet_x / float(arena_x))),   mapping(complexity, float(bot_bullet_y / float(arena_y))),   mapping(complexity, abs(float(agent_x - bot_x)) / float(arena_x)),   mapping(complexity, abs(float(agent_y - bot_y)) / float(arena_y))]]) bot_take_action() if np.random.rand(1) < e or i <= 5: a = random.randint(0, 8) else: a, _ = sess.run([predict, Q],feed_dict={input_layer : current_state}) r, c, d, winner = action(a + 1, bot_action) next_state = np.array([[mapping(complexity, float(agent_x) / float(arena_x)),   mapping(complexity, float(agent_y) / float(arena_y)),   mapping(complexity, float(bot_x) / float(arena_x)),   mapping(complexity, float(bot_y) / float(arena_y)),   mapping(complexity, float(bot_bullet_x / float(arena_x))),   mapping(complexity, float(bot_bullet_y / float(arena_y))),   mapping(complexity, abs(float(agent_x - bot_x)) / float(arena_x)),   mapping(complexity, abs(float(agent_y - bot_y)) / float(arena_y))]]) samples.append([current_state, a, r, next_state]) if len(samples) > 10: for count in xrange(batch_size): [batch_current_state, action_taken, reward, batch_next_state] = samples[random.randint(0, len(samples) - 1)] batch_allQ = sess.run(Q, feed_dict={input_layer : batch_current_state}) batch_Q1 = sess.run(Q, feed_dict = {input_layer : batch_next_state}) batch_maxQ1 = np.max(batch_Q1) batch_targetQ = batch_allQ batch_targetQ[0][a] = reward + y * batch_maxQ1 sess.run([updateModel], feed_dict={input_layer : batch_current_state, next_Q : batch_targetQ}) rAll += r if d == True: e = 1. / ((i / 50) + 10) success += 1 break display.update() rList.append(rAll) print winner  print "Successful episodes: %d out of %d. Success Rate = %d" % (success, num_episodes, float(success)/float(num_episodes)) plt.plot(rList)  plt.show()