Last active
July 28, 2023 01:39
-
-
Save riponbanik/85f477d630b8008fbcf6118e12314464 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Description: CloudFormation template to create AWS Lake Formation workshop resources | |
| Metadata: | |
| 'AWS::CloudFormation::Interface': | |
| ParameterGroups: | |
| - Label: | |
| default: Database Configuration | |
| Parameters: | |
| - TPCDBName | |
| - DBMasterUser | |
| - DBMasterPassword | |
| - Label: | |
| default: Misc Configuration | |
| Parameters: | |
| - EEKeyPair | |
| - LatestAmiId | |
| ParameterLabels: | |
| TPCDBName: | |
| default: Database Name | |
| DBMasterUser: | |
| default: Master Username | |
| DBMasterPassword: | |
| default: Master User Password | |
| EEKeyPair: | |
| default: EC2 Key Pair | |
| LatestAmiId: | |
| default: Latest AMI Id | |
| Mappings: | |
| SubnetConfig: | |
| VPC: | |
| CIDR: 10.0.0.0/16 | |
| PublicOne: | |
| CIDR: 10.0.0.0/24 | |
| PublicTwo: | |
| CIDR: 10.0.1.0/24 | |
| Private: | |
| CIDR: 10.0.2.0/24 | |
| Constants: | |
| EC2InstanceType: | |
| Name: t2.small | |
| AutoHibernateTimeout: | |
| Name: 30 | |
| Parameters: | |
| TPCDBName: | |
| Type: String | |
| Default: tpc | |
| AllowedValues: | |
| - tpc | |
| Description: Name of the database that will be created. | |
| DBMasterUser: | |
| Type: String | |
| Default: tpcadmin | |
| Description: Master username for TPC database. | |
| DBMasterPassword: | |
| Type: String | |
| Default: BigData26! | |
| Description: Master password for TPC database. | |
| EEKeyPair: | |
| Description: Amazon EC2 Key Pair | |
| Type: 'AWS::EC2::KeyPair::KeyName' | |
| MinLength: 1 | |
| LatestAmiId: | |
| Type: 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>' | |
| Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 | |
| AllowedValues: | |
| - /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 | |
| Description: Image ID for the EC2 helper instance. DO NOT change this. | |
| Resources: | |
| GlueServiceRole: | |
| Type: 'AWS::IAM::Role' | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Principal: | |
| Service: | |
| - glue.amazonaws.com | |
| - lakeformation.amazonaws.com | |
| - firehose.amazonaws.com | |
| Action: 'sts:AssumeRole' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole' | |
| - 'arn:aws:iam::aws:policy/AmazonKinesisFullAccess' | |
| Policies: | |
| - PolicyName: LF-Data-Lake-Storage-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| - /* | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| - PolicyName: Glue-Demo-Access-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - /* | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - PolicyName: LF-DataAccess-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'lakeformation:GetDataAccess' | |
| - 'lakeformation:GrantPermissions' | |
| Resource: '*' | |
| - PolicyName: LF-Workflow-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'iam:PassRole' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:iam::' | |
| - !Ref 'AWS::AccountId' | |
| - ':role/LF-GlueServiceRole' | |
| - !Join | |
| - '' | |
| - - 'arn:aws:iam::' | |
| - !Ref 'AWS::AccountId' | |
| - ':role/LakeFormationWorkflowRole' | |
| RoleName: LF-GlueServiceRole | |
| DataAnalystGlueServiceRole: | |
| Type: 'AWS::IAM::Role' | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Principal: | |
| Service: | |
| - glue.amazonaws.com | |
| - lakeformation.amazonaws.com | |
| - firehose.amazonaws.com | |
| Action: 'sts:AssumeRole' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole' | |
| - 'arn:aws:iam::aws:policy/AmazonKinesisFullAccess' | |
| Policies: | |
| - PolicyName: DA-Data-Lake-Storage-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| - /* | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| - PolicyName: DA-Glue-Access-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - /* | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - PolicyName: DA-DataAccess-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'lakeformation:GetDataAccess' | |
| - 'lakeformation:GrantPermissions' | |
| Resource: '*' | |
| - PolicyName: DA-Workflow-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'iam:PassRole' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:iam::' | |
| - !Ref 'AWS::AccountId' | |
| - ':role/DA-GlueServiceRole' | |
| - !Join | |
| - '' | |
| - - 'arn:aws:iam::' | |
| - !Ref 'AWS::AccountId' | |
| - ':role/LakeFormationWorkflowRole' | |
| RoleName: DA-GlueServiceRole | |
| DataEngineerGlueServiceRole: | |
| Type: 'AWS::IAM::Role' | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Principal: | |
| Service: | |
| - glue.amazonaws.com | |
| - lakeformation.amazonaws.com | |
| - firehose.amazonaws.com | |
| Action: 'sts:AssumeRole' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole' | |
| - 'arn:aws:iam::aws:policy/AmazonKinesisFullAccess' | |
| Policies: | |
| - PolicyName: DE-Data-Lake-Storage-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| - /* | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| - PolicyName: DE-Glue-Access-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - /* | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - PolicyName: DE-DataAccess-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'lakeformation:GetDataAccess' | |
| - 'lakeformation:GrantPermissions' | |
| Resource: '*' | |
| - PolicyName: DA-Workflow-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'iam:PassRole' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:iam::' | |
| - !Ref 'AWS::AccountId' | |
| - ':role/DE-GlueServiceRole' | |
| - !Join | |
| - '' | |
| - - 'arn:aws:iam::' | |
| - !Ref 'AWS::AccountId' | |
| - ':role/LakeFormationWorkflowRole' | |
| RoleName: DE-GlueServiceRole | |
| AmazonKinesisFirehoseFullAccess: | |
| Type: 'AWS::IAM::Role' | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Principal: | |
| Service: | |
| - firehose.amazonaws.com | |
| Action: 'sts:AssumeRole' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/AmazonKinesisFirehoseFullAccess' | |
| Policies: | |
| - PolicyName: LF-Stream-Data-Storage-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| - /* | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref DataLakeBucket | |
| RoleName: LF-KinesisServiceRole | |
| DataLakeBucket: | |
| Type: 'AWS::S3::Bucket' | |
| Properties: | |
| OwnershipControls: | |
| Rules: | |
| - ObjectOwnership: BucketOwnerPreferred | |
| BucketName: !Join | |
| - '-' | |
| - - lf-data-lake | |
| - !Ref 'AWS::AccountId' | |
| LFWorkshopBucket: | |
| Type: 'AWS::S3::Bucket' | |
| Properties: | |
| OwnershipControls: | |
| Rules: | |
| - ObjectOwnership: BucketOwnerPreferred | |
| BucketName: !Join | |
| - '-' | |
| - - lf-workshop | |
| - !Ref 'AWS::AccountId' | |
| LFUsersPassword: | |
| Type: 'AWS::SecretsManager::Secret' | |
| Properties: | |
| Description: Secret password for all workshop users | |
| Name: !Sub '${AWS::StackName}-lf-users-credentials' | |
| GenerateSecretString: | |
| SecretStringTemplate: '{"username":"all-lf-users"}' | |
| GenerateStringKey: password | |
| PasswordLength: 16 | |
| ExcludeCharacters: '"@/\' | |
| ADFSUsersPassword: | |
| Type: 'AWS::SecretsManager::Secret' | |
| Properties: | |
| Description: Secret password for all ADFS users | |
| Name: !Sub '${AWS::StackName}-adfs-users-credentials' | |
| SecretString: '{"username":"all-windows-users","password":"Password1!"}' | |
| DataAdminUser: | |
| Type: 'AWS::IAM::User' | |
| Properties: | |
| Path: / | |
| LoginProfile: | |
| Password: !Sub '{{resolve:secretsmanager:${LFUsersPassword}::password}}' | |
| PasswordResetRequired: false | |
| Policies: | |
| - PolicyName: LF-DataLake-Admin-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: 'iam:CreateServiceLinkedRole' | |
| Resource: '*' | |
| Condition: | |
| StringEquals: | |
| 'iam:AWSServiceName': lakeformation.amazonaws.com | |
| - Effect: Allow | |
| Action: | |
| - 'iam:PutRolePolicy' | |
| Resource: !Join | |
| - '' | |
| - - 'arn:aws:iam::' | |
| - !Ref 'AWS::AccountId' | |
| - >- | |
| :role/aws-service-role/lakeformation.amazonaws.com/AWSServiceRoleForLakeFormationDataAccess | |
| - Effect: Allow | |
| Action: 'iam:PassRole' | |
| Resource: | |
| - 'arn:aws:iam::*:role/LF-GlueServiceRole' | |
| - PolicyName: LF-DataLake-Admin-RAM-Invitation-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'ram:AcceptResourceShareInvitation' | |
| - 'ram:RejectResourceShareInvitation' | |
| - 'ec2:DescribeAvailabilityZones' | |
| - 'ram:EnableSharingWithAwsOrganization' | |
| Resource: '*' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/AWSLakeFormationDataAdmin' | |
| - 'arn:aws:iam::aws:policy/AWSGlueConsoleFullAccess' | |
| - 'arn:aws:iam::aws:policy/AWSLakeFormationCrossAccountManager' | |
| UserName: lf-data-admin | |
| DataScientistUser: | |
| Type: 'AWS::IAM::User' | |
| Properties: | |
| Path: / | |
| LoginProfile: | |
| Password: !Sub '{{resolve:secretsmanager:${LFUsersPassword}::password}}' | |
| PasswordResetRequired: false | |
| Policies: | |
| - PolicyName: LF-Athena-Query-Result-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:Put*' | |
| - 's3:Get*' | |
| - 's3:List*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - /athena-results/* | |
| - PolicyName: LF-Cell-Level-Filter-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'lakeformation:StartQueryPlanning' | |
| - 'lakeformation:GetQueryState' | |
| - 'lakeformation:GetWorkUnits' | |
| - 'lakeformation:GetWorkUnitResults' | |
| Resource: '*' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/AmazonAthenaFullAccess' | |
| - 'arn:aws:iam::aws:policy/AmazonSageMakerFullAccess' | |
| UserName: lf-data-scientist | |
| DataAnalystUser: | |
| Type: 'AWS::IAM::User' | |
| Properties: | |
| Path: / | |
| LoginProfile: | |
| Password: !Sub '{{resolve:secretsmanager:${LFUsersPassword}::password}}' | |
| PasswordResetRequired: false | |
| Policies: | |
| - PolicyName: LF-Athena-Query-Result-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:Put*' | |
| - 's3:Get*' | |
| - 's3:List*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - /athena-results/* | |
| - PolicyName: LF-Athena-Run-Tagging-Query | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 'lakeformation:StartQueryPlanning' | |
| - 'lakeformation:GetQueryState' | |
| - 'lakeformation:GetWorkUnits' | |
| - 'lakeformation:GetWorkUnitResults' | |
| Resource: "*" | |
| - PolicyName: LF-PassRole | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: 'iam:PassRole' | |
| Resource: | |
| - 'arn:aws:iam::*:role/DA-GlueServiceRole' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/AmazonAthenaFullAccess' | |
| - 'arn:aws:iam::aws:policy/AWSGlueConsoleFullAccess' | |
| - 'arn:aws:iam::aws:policy/AmazonRedshiftDataFullAccess' | |
| - 'arn:aws:iam::aws:policy/AmazonRedshiftFullAccess' | |
| - 'arn:aws:iam::aws:policy/AmazonRedshiftQueryEditorV2FullAccess' | |
| UserName: lf-data-analyst | |
| DataEngineerUser: | |
| Type: 'AWS::IAM::User' | |
| Properties: | |
| Path: / | |
| LoginProfile: | |
| Password: !Sub '{{resolve:secretsmanager:${LFUsersPassword}::password}}' | |
| PasswordResetRequired: false | |
| Policies: | |
| - PolicyName: LF-Athena-Query-Result-Policy | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: | |
| - 's3:Put*' | |
| - 's3:Get*' | |
| - 's3:List*' | |
| Resource: | |
| - !Join | |
| - '' | |
| - - 'arn:aws:s3:::' | |
| - !Ref LFWorkshopBucket | |
| - /athena-results/* | |
| - PolicyName: LF-PassRole | |
| PolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Action: 'iam:PassRole' | |
| Resource: | |
| - 'arn:aws:iam::*:role/DE-GlueServiceRole' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/AmazonAthenaFullAccess' | |
| - 'arn:aws:iam::aws:policy/AWSGlueConsoleFullAccess' | |
| UserName: lf-data-engineer | |
| EC2Role: | |
| Type: 'AWS::IAM::Role' | |
| Properties: | |
| Path: / | |
| RoleName: EC2Role | |
| AssumeRolePolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Principal: | |
| Service: | |
| - ec2.amazonaws.com | |
| Action: 'sts:AssumeRole' | |
| ManagedPolicyArns: | |
| - 'arn:aws:iam::aws:policy/AdministratorAccess' | |
| GlueCrawler: | |
| Type: 'AWS::Glue::Crawler' | |
| Properties: | |
| DatabaseName: tpc | |
| Description: AWS Glue Crawler to crawl parquet data | |
| Name: TPC Crawler | |
| Role: !GetAtt | |
| - GlueServiceRole | |
| - Arn | |
| Targets: | |
| S3Targets: | |
| - Path: !Join | |
| - '' | |
| - - !Ref DataLakeBucket | |
| - /tpcparquet/ | |
| SchemaChangePolicy: | |
| UpdateBehavior: UPDATE_IN_DATABASE | |
| DeleteBehavior: LOG | |
| AWSCloud9SSMAccessRole: | |
| Type: AWS::IAM::Role | |
| Properties: | |
| AssumeRolePolicyDocument: | |
| Version: 2012-10-17 | |
| Statement: | |
| - Effect: Allow | |
| Principal: | |
| Service: | |
| - cloud9.amazonaws.com | |
| - ec2.amazonaws.com | |
| Action: | |
| - 'sts:AssumeRole' | |
| Description: 'Service linked role for AWS Cloud9' | |
| Path: '/service-role/' | |
| ManagedPolicyArns: | |
| - arn:aws:iam::aws:policy/AWSCloud9SSMInstanceProfile | |
| RoleName: 'AWSCloud9SSMAccessRole' | |
| AWSCloud9SSMInstanceProfile: | |
| Type: "AWS::IAM::InstanceProfile" | |
| Properties: | |
| InstanceProfileName: AWSCloud9SSMInstanceProfile | |
| Path: "/cloud9/" | |
| Roles: | |
| - | |
| Ref: AWSCloud9SSMAccessRole | |
| Outputs: | |
| LFDataLakeBucketName: | |
| Description: Lake Formation Data Lake Bucket Name | |
| Value: !Ref DataLakeBucket | |
| LFWorkshopBucketName: | |
| Description: Lake Formation Workshop Bucket Name | |
| Value: !Ref LFWorkshopBucket | |
| AthenaQueryResultLocation: | |
| Description: Athena Query Result Location | |
| Value: !Join | |
| - '' | |
| - - 's3://' | |
| - !Ref LFWorkshopBucket | |
| - /athena-results/ | |
| MetadataLocation: | |
| Description: Metadata Location | |
| Value: !Join | |
| - '' | |
| - - 's3://' | |
| - !Ref LFWorkshopBucket | |
| - /metadata | |
| LFUsersCredentials: | |
| Description: AWS Secrets Manager Secret Name for all workshop users credentials | |
| Value: !Sub >- | |
| https://${AWS::Region}.console.aws.amazon.com/secretsmanager/secret?name=${AWS::StackName}-lf-users-credentials | |
| ADFSUsersCredentials: | |
| Description: AWS Secrets Manager Secret Name for all ADFS users credentials | |
| Value: !Sub >- | |
| https://${AWS::Region}.console.aws.amazon.com/secretsmanager/secret?name=${AWS::StackName}-adfs-users-credentials | |
| ConsoleIAMLoginUrl: | |
| Description: Console IAM Login URL to try out different users | |
| Value: !Join | |
| - '' | |
| - - 'https://' | |
| - !Ref 'AWS::AccountId' | |
| - .signin.aws.amazon.com/console |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment