screenshot.ts

screenshot.ts

import Anthropic from '@anthropic-ai/sdk';

const anthropic = new Anthropic();

const system_prompt = `
You are using a Mac running MacOS 15.1.1 with internet access.
You are a helpful assistant that can help with tasks on a computer.
`

async function takeKindleScreenshot(anthropic: Anthropic, pageNum: number) {
  const message = await anthropic.beta.messages.create({
    model: "claude-3-5-sonnet-20241022",
    max_tokens: 1024,
    tools: [
      {
        type: "computer_20241022",
        name: "computer",
        display_width_px: 1024,
        display_height_px: 768,
        display_number: 1
      }
    ],
    messages: [{ 
      role: "user", 
      content: pageNum === 0 
        ? "Take a screenshot of the Kindle app window and save it to the desktop as 'kindle_page_0.png', then click the right side of the window to go to the next page" 
        : `The previous page was ${pageNum}. Take a screenshot of the Kindle app window and save it to the desktop as 'kindle_page_${pageNum}.png', then click the right side to go to the next page. If you notice we're at the end of the book, let me know.`
    }],
    betas: ["computer-use-2024-10-22"],
    system: system_prompt
  });

  // Debug message structure
  console.log('Message structure:');
  console.log('Type:', typeof message);
  console.log('Keys:', Object.keys(message));
  console.log('\nContent array:');
  message.content.forEach((item, index) => {
    console.log(`\nContent item ${index}:`);
    console.log('Type:', item.type);
    console.log('Keys:', Object.keys(item));
    console.log('Full item:', JSON.stringify(item, null, 2));
  });

  // Original screenshot saving logic
  for (const content of message.content) {

    if (content.type === 'tool_use') {
      console.log('content is tool use', content);
    }

    if (content.type === 'tool_use' && 'screenshot' in content && typeof content.screenshot === 'string') {
      const base64Data = content.screenshot;
      const buffer = Buffer.from(base64Data, 'base64');
      const fs = require('fs');
      fs.writeFileSync(`~/Desktop/claude_view_${pageNum}.png`, buffer);
      console.log(`Saved Claude's view for page ${pageNum}`);
    }
  }

  return message;
}

async function main() {
  let pageNum = 0;
  
  while (true) {
    try {
      console.log(`Processing page ${pageNum}...`);
      const response = await takeKindleScreenshot(anthropic, pageNum);
      
      // Check if we've reached the end of the book
      const lastMessage = response.content[response.content.length - 1];
      if (lastMessage.type === 'text' && 
          lastMessage.text.toLowerCase().includes('end of the book')) {
        console.log('Reached end of book');
        break;
      }

      pageNum++;
      // Wait a moment for the page turn animation
      await new Promise(resolve => setTimeout(resolve, 2000));
    } catch (error) {
      console.error('Error processing page:', error);
      if (error instanceof Error) {
        console.error('Error details:', error.message);
        console.error('Stack trace:', error.stack);
      }
      // Add a delay before retrying after an error
      await new Promise(resolve => setTimeout(resolve, 5000));
    }
  }
}

main().catch(console.error);