deno-parsing-indeed.com.ts

import { DOMParser } from "https://deno.land/x/deno_dom/deno-dom-wasm.ts";

const response = await fetch("https://be.indeed.com/jobs?q=Marketing&lang=en");
const reply = await response.text();

const doc = new DOMParser().parseFromString(reply, "text/html")!;

const jobKeys = doc
  .getElementsByClassName("jobtitle")
  .map((jobNode) => jobNode.getAttribute("href"))
  .map((jobLink) => jobLink?.split("?")[1])
  .map((jobParam) => new URLSearchParams(jobParam).get("jk"));

const fetchingJobs = jobKeys.map(async (jobKey, i) => {
  await randomSleep(i * 1000);
  const response = await fetch(
    `https://be.indeed.com/viewjob?jk=${jobKey}&from=vjs&vjs=1`,
    {
      headers: {
        "user-agent": "bot parsing",
      },
    }
  );
  const reply = await response.json();

  const {
    jobTitle: title,
    vfvm: { jobSource: company, viewApplyJobLink },
    sicm: { cmL: companyLogo },
    jobLocation: location,
    timestamp,
  } = reply;

  return {
    title,
    company,
    viewApplyJobLink,
    companyLogo,
    location,
    timestamp,
    jobKey,
  };
});

const jobs = await Promise.all(fetchingJobs);

const jobDescrUrl = `https://be.indeed.com/rpc/jobdescs?jks=${encodeURIComponent(
  jobKeys.join(",")
)}`;
const descrResponse = await fetch(jobDescrUrl);
const descrReply = await descrResponse.json();

const jobsWithDescr = jobs.map((job) => ({
  ...job,
  description: descrReply[job?.jobKey || ""],
}));

console.log({ jobsWithDescr });

async function randomSleep(min: number) {
  const delay = min + ~~(Math.random() * 1000);
  console.log(`I'm sleeping for ${delay}ms`);
  await new Promise((resolve) => setTimeout(resolve, delay));
}