iht

# See previous gist https://gist.github.com/iht/d5bedff9ee86ed5223cfa2ad06080ca0
import json 
# ...
from langchain.output_parsers import ResponseSchema, StructuredOutputParser

def _dict_to_json(x: dict) -> str:
  return "```\n" + json.dumps(x) + "\n```"

response_schema = [
    ResponseSchema(name="query", description="SQL query to solve the user question."),

iht

from operator import itemgetter

from langchain.chat_models import ChatVertexAI
from langchain.prompts import PromptTemplate
from langchain.schema import StrOutputParser
from langchain.schema.vectorstore import VectorStoreRetriever

## Chain1 
# The input is {"input": "Some question written by a user"}
some_model1 =  ChatVertexAI(model_name="codechat-bison", max_output_tokens=2048)

iht

# Google Cloud zone to use for the resources
ZONE=europe-west1-b

# If not running in Cloud Shell, set the variable GOOGLE_CLOUD_PROJECT to your project id
# GOOGLE_CLOUD_PROJECT=<PROJECT ID>

# Get service account id
SERVICE_ACCOUNT=$(gcloud iam service-accounts list | grep "[email protected]" | cut -d " " -f 2)

# Add permission to run Dataflow jobs

iht

package dev.herraiz.beam.schemas;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.client.googleapis.util.Utils;
import com.google.api.client.json.JsonFactory;
import com.google.api.services.bigquery.model.TableSchema;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryUtils;
import org.apache.beam.sdk.schemas.Schema;

iht

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;


public static void main(String[] args) {
  PipelineOptions opts =
    PipelineOptionsFactory.fromArgs(args);

  DataflowPipelineOptions dataflowPipelineOptions =
    opts.as(DataflowPipelineOptions.class);

iht

def run_pipeline(argv):
    opts: PipelineOptions = PipelineOptions(argv)
    gcloud_opts: GoogleCloudOptions = opts.view_as(GoogleCloudOptions)
   
    if opts.i_want_streaming_engine:
      gcloud_opts.enable_streaming_engine = True
    else:
      gcloud_opts.enable_streaming_engine = False
    
    ...

iht

resource "google_dataflow_flex_template_job" "big_data_job" {
  provider                = google-beta
  name                    = "dataflow-flextemplates-job"
  container_spec_gcs_path = "gs://my-bucket/templates/template.json"
  parameters = {
    i_want_streaming_engine = true
  }
}

iht

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.options.PipelineOptionsFactory;


public static void main(String[] args) {
  PipelineOptions opts =
    PipelineOptionsFactory.fromArgs(args);

  DataflowPipelineOptions dataflowPipelineOptions =
    opts.as(DataflowPipelineOptions.class);

iht

def run_pipeline(argv):
    opts: PipelineOptions = PipelineOptions(argv)
    gcloud_opts: GoogleCloudOptions = opts.view_as(GoogleCloudOptions)

    gcloud_opts.enable_streaming_engine = True
    gcloud_opts.job_name = "Overwrite the name of your job"
    
    ...

iht

def find_external_functions(key, stats):
  output = {}
  for k,v in stats.items():
    _, _, _, _, d = v
    if key in d:
      output[k] = v

  return output