find-similar-ngrams.sql

CREATE TEMP FUNCTION term_years(term STRING)
RETURNS ARRAY<STRUCT<year INT64, term_frequency INT64>> AS (
  (
      SELECT years
      FROM `gcp-pdp-words-dev.sandbox.eng_bert_preprocessed_year_term_frequencies`
      WHERE preprocessed_term = term
  )
  --   SELECT ARRAY_AGG(STRUCT<year INT64, term_frequency INT64>(z.year, z.term_frequency + IFNULL(t.term_frequency, 0)))
  --   FROM 
  --   (
  --     SELECT year, 0 AS term_frequency
  --     FROM 
  --     (
  --       SELECT preprocessed_term, GENERATE_ARRAY(MIN(year), MAX(year)) AS year
  --       FROM
  --       (
  --         SELECT preprocessed_term, years.year, years.term_frequency
  --         FROM `gcp-pdp-words-dev.sandbox.eng_bert_preprocessed_year_term_frequencies` JOIN UNNEST(years) AS years
  --         WHERE preprocessed_term = term
  --       )
  --       GROUP BY preprocessed_term
  --     ) JOIN UNNEST(year) AS year
  --   ) AS z LEFT JOIN
  --   (
  --     SELECT preprocessed_term, years.year, years.term_frequency
  --     FROM `gcp-pdp-words-dev.sandbox.eng_bert_preprocessed_year_term_frequencies` JOIN UNNEST(years) AS years
  --     WHERE preprocessed_term = term
  --   ) AS t
  --   ON (z.year = t.year)
  -- )
);

WITH 
term AS (SELECT 'maga' AS term),
expanded AS (SELECT (SELECT term.term FROM term) AS term, year, term_frequency FROM (SELECT term_years('make america great again') AS x) JOIN UNNEST(x) AS y),
maga AS (SELECT (SELECT term.term FROM term) AS term, year, term_frequency FROM (SELECT term_years((SELECT term.term FROM term)) AS x) JOIN UNNEST(x) AS y)

SELECT CORR(maga.term_frequency, expanded.term_frequency)
FROM term, maga LEFT JOIN expanded ON maga.year = expanded.year