rajathithan

import os
import pickle
from googleapiclient.discovery import build
from googleapiclient.http import MediaIoBaseDownload
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request

# ID of the folder to be downloaded.
# ID can be obtained from the URL of the folder
FOLDER_ID = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"  # an example folder

rajathithan

""" 
Purpose     : To retrieve details on cloud run services across all projects under an org
Author      : Rajathithan Rajasekar
Date        : Jan 23, 2024
"""

from googleapiclient.discovery import build
import google.auth
import csv

rajathithan

import apache_beam as beam

from log_elements import LogElements

lines = [
    "apple orange grape banana apple banana",
    "banana orange banana papaya"
]

with beam.Pipeline() as p:

rajathithan

import apache_beam as beam
from generate_event import GenerateEvent
from apache_beam.transforms.window import FixedWindows
from apache_beam.transforms.trigger import AfterWatermark
from apache_beam.transforms.trigger import AfterCount
from apache_beam.transforms.trigger import AccumulationMode
from apache_beam.utils.timestamp import Duration
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options import StandardOptions
from log_elements import LogElements

rajathithan

import apache_beam as beam
from apache_beam.testing.test_stream import TestStream
from datetime import datetime
import pytz


class GenerateEvent(beam.PTransform):

    @staticmethod
    def sample_data():

rajathithan

from apache_beam.transforms.trigger import AccumulationMode
from apache_beam.utils.timestamp import Duration
from log_elements import LogElements


class CountEventsWithEarlyTrigger(beam.PTransform):
  def expand(self, events):
    return (events
            | beam.WindowInto(FixedWindows(1 * 24 * 60 * 60),  # 1 Day Window
                              trigger=AfterWatermark(early=AfterCount(1)),

rajathithan

#!/bin/bash
# Installation of Cloud-sql-proxy in DataProc clusters
# Script Re-Modified by - Rajathithan Rajasekar
# Date - October 12, 2023
# projectname:region-name:instance-name
# Reference taken from - https://github.com/GoogleCloudDataproc/initialization-actions/blob/master/cloud-sql-proxy/cloud-sql-proxy.sh 

# Readonly Variables
readonly PROXY_DIR='/var/run/cloud_sql_proxy'
readonly PROXY_BIN='/usr/local/bin/cloud_sql_proxy'

rajathithan

# Cloud Function code to trigger dataflow flex template
# Author: Rajathithan Rajasekar 
# Version : 1.0
# Date: 08/05/2023

from __future__ import annotations
from typing import Any
import google.auth
from google.auth.transport.requests import AuthorizedSession
import requests

rajathithan

# Dataflow Flex template job to upload xml data from GCS to BQ
# Author: Rajathithan Rajasekar 
# Version : 1.0
# Date: 08/05/2023

import sys
import json
import logging
import argparse
import xmltodict

rajathithan

# Dataflow Job to create custom dataflow template for upload of xml data from GCS to BQ
# Author: Rajathithan Rajasekar 
# Version : 1.0
# Date: 08/05/2023


import sys
import json
import logging
import argparse