sample_streaming.js

// A simple example of streaming data in JSON format from ClickHouse. This is not performance optimized and doesn't 
// handle errors or edge cases such as large single line payloads

const credentials = btoa('play:');
const response = await fetch('https://clickpy-clickhouse.clickhouse.com', {
    method: 'POST',
    body: 'SELECT * FROM pypi.pypi_downloads LIMIT 10000000 FORMAT JSONEachRow',
    headers: {
      'Authorization': `Basic ${credentials}`,
      'Content-Type': 'application/x-www-form-urlencoded'
    }
  });

  const reader = response.body.getReader();
  const decoder = new TextDecoder();
  let partialData = '';
  let c = 0;
  while (true) {
    const { done, value } = await reader.read();
    if (done) break;
    partialData += decoder.decode(value, { stream: true });
    const lines = partialData.split('\n'); // Process all but the last line (it might be incomplete, so save it for the next loop)
    for (let i = 0; i < lines.length - 1; i++) {
      if (lines[i].trim()) {
        const jsonObject = JSON.parse(lines[i]);
        //console.log(jsonObject);
        c+=1;
      }
    }
    // Save the last line (incomplete row) to process in the next loop iteration
    partialData = lines[lines.length - 1];
  }
  // Process the remaining partial data if it forms a complete line
  if (partialData.trim()) {
    const jsonObject = JSON.parse(partialData);
    c+=1;
  }
  console.log(c);