Reinforcement Learning for Automated Exploration

class RNDCuriosityModule:
    """Computes a reward based on how 'surprised' we are by a frame."""
    def __init__(self, device='cuda'):
        # Alice: Fixed random network (the 'Target')
        self.random_net = RandomNetwork().to(device)
        # Bob: Trainable network (the 'Predictor')
        self.predictor_net = PredictorNetwork().to(device)

    def compute_intrinsic_reward(self, frames):
        with torch.no_grad():
            target_features = self.random_net(frames)

        predicted_features = self.predictor_net(frames)

        # The more Bob struggles to predict Alice's output,
        # the higher the 'curiosity' reward.
        reward = torch.mean((predicted_features - target_features) ** 2, dim=1)
        return reward

class ActionSpace:
    """Discrete actions the agent can take (e.g., Click X,Y or Press Key)."""
    def action_to_command(self, action_id):
        if action_id < self.num_click_actions:
            # Map grid cell to screen pixel coordinates
            return {'type': 'click', 'x': px, 'y': py}
        elif action_id < self.total_actions:
            # Map to specific keyboard keys (ESC, Space, etc.)
            return {'type': 'key', 'key': key_map[action_id - offset]}
        return {'type': 'wait'}

def train_explorer():
    # 1. Initialize our curiosity module and the agent
    curiosity = RNDCuriosityModule()
    agent = PPOAgent()

    while True:
        # 2. Agent interacts with the Vulkan app
        action = agent.select_action(current_screen)
        execute_in_vulkan(action)

        # 3. Calculate 'Surprise' reward
        reward = curiosity.compute_intrinsic_reward(next_screen)

        # 4. If the app crashed, give a huge bonus reward!
        if app_crashed(): reward += 10.0

        # 5. Learn from this experience
        agent.update(experience_buffer)
        curiosity.update(next_screen)

name: RL Exploration Testing

on:
  schedule:
    - cron: '0 2 * * *'  # Run nightly at 2 AM

jobs:
  explore:
    runs-on: ubuntu-latest
    timeout-minutes: 360  # 6 hours

    steps:
      - uses: actions/checkout@v3

      - name: Build application
        run: |
          mkdir build && cd build
          cmake .. -DCMAKE_BUILD_TYPE=Release
          make -j$(nproc)

      - name: Install RL dependencies
        run: |
          pip install torch torchvision numpy opencv-python pyautogui

      - name: Run RL exploration
        run: |
          python scripts/rl_explorer.py \
            --app ./build/my_vulkan_app \
            --episodes 1000 \
            --checkpoint models/explorer_checkpoint.pt

      - name: Upload crash logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: rl-discovered-crashes
          path: crashes/

class BenchmarkingAgent(PPOAgent):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.fps_history = []

    def compute_reward(self, frame, fps):
        """Reward agent for finding slow scenes"""
        # Intrinsic curiosity reward
        intrinsic = self.curiosity.compute_intrinsic_reward(frame)

        # Extrinsic reward: finding low FPS is good (for benchmarking)
        extrinsic = 0.0
        if fps < 30:
            extrinsic = (30 - fps) / 30.0  # Reward proportional to slowness

        return intrinsic + extrinsic

# Reward function for impressive demos
def demo_reward(state, action, next_state):
    # High reward for visual variety
    visual_change = np.mean(np.abs(next_state - state))

    # High reward for accessing different menus
    ui_diversity = measure_ui_diversity(next_state)

    # Penalty for getting stuck
    stuck_penalty = -1.0 if visual_change < 0.01 else 0.0

    return visual_change + ui_diversity + stuck_penalty

import matplotlib.pyplot as plt
from collections import deque

class ExplorationMonitor:
    def __init__(self):
        self.visited_states = {}  # Hash -> visit count
        self.crashes_found = []
        self.episode_rewards = deque(maxlen=100)

    def record_state(self, state):
        state_hash = hash(state.tobytes())

        if state_hash not in self.visited_states:
            self.visited_states[state_hash] = 0

        self.visited_states[state_hash] += 1

    def plot_exploration(self):
        # Histogram of state visit counts
        visit_counts = list(self.visited_states.values())

        plt.figure(figsize=(10, 5))
        plt.subplot(1, 2, 1)
        plt.hist(visit_counts, bins=50)
        plt.xlabel('Visit Count')
        plt.ylabel('Number of States')
        plt.title('State Visit Distribution')

        # Reward over time
        plt.subplot(1, 2, 2)
        plt.plot(self.episode_rewards)
        plt.xlabel('Episode')
        plt.ylabel('Total Reward')
        plt.title('Learning Progress')

        plt.tight_layout()
        plt.savefig('exploration_stats.png')

    def coverage_stats(self):
        total_states = len(self.visited_states)
        novel_states = sum(1 for c in self.visited_states.values() if c == 1)

        print(f"Total unique states: {total_states}")
        print(f"Novel states (seen once): {novel_states}")
        print(f"Crashes found: {len(self.crashes_found)}")