# Copyright 2019 Jonathan Eskew

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#   http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: "Serverless client-side telemetry aggregation infrastructure"

Resources:
  # Telemetry events will be ingested through a CloudFront distribution
  ClientSideTelemetryIngester:
    Type: AWS::CloudFront::Distribution
    Properties:
      DistributionConfig:
        Enabled: True
        HttpVersion: http2
        DefaultCacheBehavior:
          AllowedMethods:
            - GET
            - HEAD
            - OPTIONS
            - PUT
            - PATCH
            - POST
            - DELETE
          ForwardedValues:
            QueryString: False
          TargetOriginId: EffOrigin
          ViewerProtocolPolicy: redirect-to-https
          # Here's where we tell CloudFront to just invoke our Lambda for every
          # request
          LambdaFunctionAssociations:
            - EventType: viewer-request
              LambdaFunctionARN:
                Ref: ClientSideTelemetryLambdaFunction.Version
        # CloudFront requires that we specify an origin, even though we won't be
        # sending any requests to it (they are all handled by the lambda
        # function).
        Origins:
          - DomainName: www.eff.org
            Id: EffOrigin
            CustomOriginConfig:
              OriginProtocolPolicy: https-only

  # This will be the lambda function that ingests individual messages
  ClientSideTelemetryLambdaFunction:
    Type: AWS::Serverless::Function
    Properties:
      Handler: 'index.handler'
      Runtime: 'nodejs8.10'
      MemorySize: 128
      Role:
        Fn::GetAtt:
          - ClientSideTelemetryLambdaRole
          - Arn
      Timeout: 5
      AutoPublishAlias: live
      InlineCode:
        Fn::Sub: |
          'use strict';
          const Firehose = require('aws-sdk/clients/firehose');

          exports.handler = (event, context, callback) => {
            const request = event.Records[0].cf.request;
            if (request.method === 'POST' && request.body && request.body.data) {
              const ecsEvent = JSON.parse(Buffer.from(request.body.data, 'base64').toString());
              if (ecsEvent['@timestamp'] === undefined) {
                ecsEvent['@timestamp'] = (new Date).toISOString();
              }

              ecsEvent['client.ip'] = request.clientIp;
              const {headers = {}} = request;
              if (ecsEvent['url.full'] === undefined && headers['referer']) {
                ecsEvent['url.full'] = headers['referer'][0].value
              }

              if (headers['user-agent']) {
                ecsEvent['user_agent.original'] = request.headers['user-agent'][0].value;
              }

              const fh = new Firehose({region: "${AWS::Region}"});
              const req = {
                DeliveryStreamName: "${ClientSideTelemetryDeliveryStream}",
                Record: {
                  Data: JSON.stringify(ecsEvent)
                }
              };

              fh.putRecord(req, (err, resp) => {
                if (err) {
                  callback(err);
                } else {
                  callback(null, {status: '200'});
                }
              });
            } else {
              callback(null, {status: '415'});
            }
          };

  # This is where our telemetry data will be stored
  ClientSideTelemetryBucket:
    Type: AWS::S3::Bucket
    Properties:
      # This will automatically move objects more than 60 days old into Glacier.
      # Omit if you want to keep telemetry in the bucket forever.
      LifecycleConfiguration:
        Rules:
          -
            Status: Enabled
            Transitions:
              -
                StorageClass: GLACIER
                TransitionInDays: 90
      PublicAccessBlockConfiguration:
        BlockPublicAcls: True
        BlockPublicPolicy: True
        RestrictPublicBuckets: True

  # The next two resources establish a schema for telemetry events
  ClientSideTelemetryDatabase:
    Type: AWS::Glue::Database
    Properties:
      # CatalogId must always be the AWS Account ID in which the database is
      # provisioned
      CatalogId:
        Ref: AWS::AccountId
      DatabaseInput:
        Description: "Aggregated client-side telemetry"
        Name: client_side_telemetry

  ClientSideEcsTelemetryTable:
    Type: AWS::Glue::Table
    Properties:
      CatalogId:
        Ref: AWS::AccountId
      DatabaseName:
        Ref: ClientSideTelemetryDatabase
      TableInput:
        Description: "Aggregated client-side Elastic Common Schema events"
        Name: ecs_events
        # This property tells Glue our data is stored outside of Glue (i.e., S3)
        TableType: EXTERNAL_TABLE
        Parameters:
          classification: parquet
        # Partition keys are present in the S3 path rather than in the file itself
        # They will allow Athena to limit the number of files it has to read to
        # execute a query when one or more partition keys is specified.
        PartitionKeys:
          -
            Type: int
            Name: year
          -
            Type: int
            Name: month
          -
            Type: int
            Name: day
          -
            Type: int
            Name: hour
        StorageDescriptor:
          Location:
            Fn::Sub: "s3://${ClientSideTelemetryBucket}/data/ecs/"
          StoredAsSubDirectories: True
          SerdeInfo:
            SerializationLibrary: 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
          InputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
          OutputFormat: 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
          # This property describes the contents of the files stored in S3
          Columns:
            - Name: ts
              Type: timestamp
              Comment: Date/time when the event originated.
            - Name: agent_ephemeral_id
              Type: string
              Comment: Ephemeral identifier of this agent.
            - Name: agent_id
              Type: string
              Comment: Unique identifier of this agent.
            - Name: agent_name
              Type: string
              Comment: Custom name of the agent.
            - Name: agent_type
              Type: string
              Comment: Type of the agent.
            - Name: agent_version
              Type: string
              Comment: Version of the agent.
            - Name: client_address
              Type: string
              Comment: Client network address.
            - Name: client_bytes
              Type: bigint
              Comment: Bytes sent from the client to the server.
            - Name: client_domain
              Type: string
              Comment: Client domain.
            - Name: client_geo_city_name
              Type: string
              Comment: City name.
            - Name: client_geo_continent_name
              Type: string
              Comment: Name of the continent.
            - Name: client_geo_country_iso_code
              Type: string
              Comment: Country ISO code.
            - Name: client_geo_country_name
              Type: string
              Comment: Country name.
            - Name: client_geo_location
              Type: struct<lat:double,lon:double>
              Comment: Longitude and latitude.
            - Name: client_geo_name
              Type: string
              Comment: User-defined description of a location.
            - Name: client_geo_region_iso_code
              Type: string
              Comment: Region ISO code.
            - Name: client_geo_region_name
              Type: string
              Comment: Region name.
            - Name: client_ip
              Type: string
              Comment: IP address of the client.
            - Name: client_mac
              Type: string
              Comment: MAC address of the client.
            - Name: client_nat_ip
              Type: string
              Comment: Client NAT ip address
            - Name: client_nat_port
              Type: bigint
              Comment: Client NAT port
            - Name: client_packets
              Type: bigint
              Comment: Packets sent from the client to the server.
            - Name: client_port
              Type: bigint
              Comment: Port of the client.
            - Name: client_user_domain
              Type: string
              Comment: Name of the directory the user is a member of.
            - Name: client_user_email
              Type: string
              Comment: User email address.
            - Name: client_user_full_name
              Type: string
              Comment: User's full name, if available.
            - Name: client_user_group_id
              Type: string
              Comment: Unique identifier for the group on the system/platform.
            - Name: client_user_group_name
              Type: string
              Comment: Name of the group.
            - Name: client_user_hash
              Type: string
              Comment: Unique user hash to correlate information for a user in anonymized form.
            - Name: client_user_id
              Type: string
              Comment: One or multiple unique identifiers of the user.
            - Name: client_user_name
              Type: string
              Comment: Short name or login of the user.
            - Name: cloud_account_id
              Type: string
              Comment: The cloud account or organization id.
            - Name: cloud_availability_zone
              Type: string
              Comment: Availability zone in which this host is running.
            - Name: cloud_instance_id
              Type: string
              Comment: Instance ID of the host machine.
            - Name: cloud_instance_name
              Type: string
              Comment: Instance name of the host machine.
            - Name: cloud_machine_type
              Type: string
              Comment: Machine type of the host machine.
            - Name: cloud_provider
              Type: string
              Comment: Name of the cloud provider.
            - Name: cloud_region
              Type: string
              Comment: Region in which this host is running.
            - Name: container_id
              Type: string
              Comment: Unique container id.
            - Name: container_image_name
              Type: string
              Comment: Name of the image the container was built on.
            - Name: container_image_tag
              Type: string
              Comment: Container image tag.
            - Name: container_labels
              Type: map<string,string>
              Comment: Image labels.
            - Name: container_name
              Type: string
              Comment: Container name.
            - Name: container_runtime
              Type: string
              Comment: Runtime managing this container.
            - Name: destination_address
              Type: string
              Comment: Destination network address.
            - Name: destination_bytes
              Type: bigint
              Comment: Bytes sent from the destination to the source.
            - Name: destination_domain
              Type: string
              Comment: Destination domain.
            - Name: destination_geo_city_name
              Type: string
              Comment: City name.
            - Name: destination_geo_continent_name
              Type: string
              Comment: Name of the continent.
            - Name: destination_geo_country_iso_code
              Type: string
              Comment: Country ISO code.
            - Name: destination_geo_country_name
              Type: string
              Comment: Country name.
            - Name: destination_geo_location
              Type: struct<lat:double,lon:double>
              Comment: Longitude and latitude.
            - Name: destination_geo_name
              Type: string
              Comment: User-defined description of a location.
            - Name: destination_geo_region_iso_code
              Type: string
              Comment: Region ISO code.
            - Name: destination_geo_region_name
              Type: string
              Comment: Region name.
            - Name: destination_ip
              Type: string
              Comment: IP address of the destination.
            - Name: destination_mac
              Type: string
              Comment: MAC address of the destination.
            - Name: destination_nat_ip
              Type: string
              Comment: Destination NAT ip
            - Name: destination_nat_port
              Type: bigint
              Comment: Destination NAT Port
            - Name: destination_packets
              Type: bigint
              Comment: Packets sent from the destination to the source.
            - Name: destination_port
              Type: bigint
              Comment: Port of the destination.
            - Name: destination_user_domain
              Type: string
              Comment: Name of the directory the user is a member of.
            - Name: destination_user_email
              Type: string
              Comment: User email address.
            - Name: destination_user_full_name
              Type: string
              Comment: User's full name, if available.
            - Name: destination_user_group_id
              Type: string
              Comment: Unique identifier for the group on the system/platform.
            - Name: destination_user_group_name
              Type: string
              Comment: Name of the group.
            - Name: destination_user_hash
              Type: string
              Comment: Unique user hash to correlate information for a user in anonymized form.
            - Name: destination_user_id
              Type: string
              Comment: One or multiple unique identifiers of the user.
            - Name: destination_user_name
              Type: string
              Comment: Short name or login of the user.
            - Name: ecs_version
              Type: string
              Comment: ECS version this event conforms to.
            - Name: error_code
              Type: string
              Comment: Error code describing the error.
            - Name: error_id
              Type: string
              Comment: Unique identifier for the error.
            - Name: error_message
              Type: string
              Comment: Error message.
            - Name: event_action
              Type: string
              Comment: The action captured by the event.
            - Name: event_category
              Type: string
              Comment: Event category.
            - Name: event_code
              Type: string
              Comment: Identification code for this event.
            - Name: event_created
              Type: timestamp
              Comment: Time when the event was first read by an agent or by your pipeline.
            - Name: event_dataset
              Type: string
              Comment: Name of the dataset.
            - Name: event_duration
              Type: bigint
              Comment: Duration of the event in nanoseconds.
            - Name: event_end
              Type: timestamp
              Comment: event.end contains the date when the event ended or when the activity was
                last observed.
            - Name: event_hash
              Type: string
              Comment: Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate
                log integrity.
            - Name: event_id
              Type: string
              Comment: Unique ID to describe the event.
            - Name: event_kind
              Type: string
              Comment: The kind of the event.
            - Name: event_module
              Type: string
              Comment: Name of the module this data is coming from.
            - Name: event_original
              Type: string
              Comment: Raw text message of entire event.
            - Name: event_outcome
              Type: string
              Comment: The outcome of the event.
            - Name: event_provider
              Type: string
              Comment: Source of the event.
            - Name: event_risk_score
              Type: float
              Comment: Risk score or priority of the event (e.g. security solutions). Use your
                system's original value here.
            - Name: event_risk_score_norm
              Type: float
              Comment: Normalized risk score or priority of the event (0-100).
            - Name: event_sequence
              Type: bigint
              Comment: Sequence number of the event.
            - Name: event_severity
              Type: bigint
              Comment: Original severity of the event.
            - Name: event_start
              Type: timestamp
              Comment: event.start contains the date when the event started or when the activity
                was first observed.
            - Name: event_timezone
              Type: string
              Comment: Event time zone.
            - Name: event_type
              Type: string
              Comment: Reserved for future usage.
            - Name: file_accessed
              Type: timestamp
              Comment: Last time the file was accessed.
            - Name: file_created
              Type: timestamp
              Comment: File creation time.
            - Name: file_ctime
              Type: timestamp
              Comment: Last time the file attributes or metadata changed.
            - Name: file_device
              Type: string
              Comment: Device that is the source of the file.
            - Name: file_directory
              Type: string
              Comment: Directory where the file is located.
            - Name: file_extension
              Type: string
              Comment: File extension.
            - Name: file_gid
              Type: string
              Comment: Primary group ID (GID) of the file.
            - Name: file_group
              Type: string
              Comment: Primary group name of the file.
            - Name: file_hash_md5
              Type: string
              Comment: MD5 hash.
            - Name: file_hash_sha1
              Type: string
              Comment: SHA1 hash.
            - Name: file_hash_sha256
              Type: string
              Comment: SHA256 hash.
            - Name: file_hash_sha512
              Type: string
              Comment: SHA512 hash.
            - Name: file_inode
              Type: string
              Comment: Inode representing the file in the filesystem.
            - Name: file_mode
              Type: string
              Comment: Mode of the file in octal representation.
            - Name: file_mtime
              Type: timestamp
              Comment: Last time the file content was modified.
            - Name: file_name
              Type: string
              Comment: Name of the file including the extension, without the directory.
            - Name: file_owner
              Type: string
              Comment: File owner's username.
            - Name: file_path
              Type: string
              Comment: Full path to the file.
            - Name: file_size
              Type: bigint
              Comment: File size in bytes.
            - Name: file_target_path
              Type: string
              Comment: Target path for symlinks.
            - Name: file_type
              Type: string
              Comment: File type (file, dir, or symlink).
            - Name: file_uid
              Type: string
              Comment: The user ID (UID) or security identifier (SID) of the file owner.
            - Name: geo_city_name
              Type: string
              Comment: City name.
            - Name: geo_continent_name
              Type: string
              Comment: Name of the continent.
            - Name: geo_country_iso_code
              Type: string
              Comment: Country ISO code.
            - Name: geo_country_name
              Type: string
              Comment: Country name.
            - Name: geo_location
              Type: struct<lat:double,lon:double>
              Comment: Longitude and latitude.
            - Name: geo_name
              Type: string
              Comment: User-defined description of a location.
            - Name: geo_region_iso_code
              Type: string
              Comment: Region ISO code.
            - Name: geo_region_name
              Type: string
              Comment: Region name.
            - Name: group_id
              Type: string
              Comment: Unique identifier for the group on the system/platform.
            - Name: group_name
              Type: string
              Comment: Name of the group.
            - Name: hash_md5
              Type: string
              Comment: MD5 hash.
            - Name: hash_sha1
              Type: string
              Comment: SHA1 hash.
            - Name: hash_sha256
              Type: string
              Comment: SHA256 hash.
            - Name: hash_sha512
              Type: string
              Comment: SHA512 hash.
            - Name: host_architecture
              Type: string
              Comment: Operating system architecture.
            - Name: host_geo_city_name
              Type: string
              Comment: City name.
            - Name: host_geo_continent_name
              Type: string
              Comment: Name of the continent.
            - Name: host_geo_country_iso_code
              Type: string
              Comment: Country ISO code.
            - Name: host_geo_country_name
              Type: string
              Comment: Country name.
            - Name: host_geo_location
              Type: struct<lat:double,lon:double>
              Comment: Longitude and latitude.
            - Name: host_geo_name
              Type: string
              Comment: User-defined description of a location.
            - Name: host_geo_region_iso_code
              Type: string
              Comment: Region ISO code.
            - Name: host_geo_region_name
              Type: string
              Comment: Region name.
            - Name: host_hostname
              Type: string
              Comment: Hostname of the host.
            - Name: host_id
              Type: string
              Comment: Unique host id.
            - Name: host_ip
              Type: string
              Comment: Host ip address.
            - Name: host_mac
              Type: string
              Comment: Host mac address.
            - Name: host_name
              Type: string
              Comment: Name of the host.
            - Name: host_os_family
              Type: string
              Comment: OS family (such as redhat, debian, freebsd, windows).
            - Name: host_os_full
              Type: string
              Comment: Operating system name, including the version or code name.
            - Name: host_os_kernel
              Type: string
              Comment: Operating system kernel version as a raw string.
            - Name: host_os_name
              Type: string
              Comment: Operating system name, without the version.
            - Name: host_os_platform
              Type: string
              Comment: Operating system platform (such centos, ubuntu, windows).
            - Name: host_os_version
              Type: string
              Comment: Operating system version as a raw string.
            - Name: host_type
              Type: string
              Comment: Type of host.
            - Name: host_uptime
              Type: bigint
              Comment: Seconds the host has been up.
            - Name: host_user_domain
              Type: string
              Comment: Name of the directory the user is a member of.
            - Name: host_user_email
              Type: string
              Comment: User email address.
            - Name: host_user_full_name
              Type: string
              Comment: User's full name, if available.
            - Name: host_user_group_id
              Type: string
              Comment: Unique identifier for the group on the system/platform.
            - Name: host_user_group_name
              Type: string
              Comment: Name of the group.
            - Name: host_user_hash
              Type: string
              Comment: Unique user hash to correlate information for a user in anonymized form.
            - Name: host_user_id
              Type: string
              Comment: One or multiple unique identifiers of the user.
            - Name: host_user_name
              Type: string
              Comment: Short name or login of the user.
            - Name: http_request_body_bytes
              Type: bigint
              Comment: Size in bytes of the request body.
            - Name: http_request_body_content
              Type: string
              Comment: The full HTTP request body.
            - Name: http_request_bytes
              Type: bigint
              Comment: Total size in bytes of the request (body and headers).
            - Name: http_request_method
              Type: string
              Comment: HTTP request method.
            - Name: http_request_referrer
              Type: string
              Comment: Referrer for this HTTP request.
            - Name: http_response_body_bytes
              Type: bigint
              Comment: Size in bytes of the response body.
            - Name: http_response_body_content
              Type: string
              Comment: The full HTTP response body.
            - Name: http_response_bytes
              Type: bigint
              Comment: Total size in bytes of the response (body and headers).
            - Name: http_response_status_code
              Type: bigint
              Comment: HTTP response status code.
            - Name: http_version
              Type: string
              Comment: HTTP version.
            - Name: labels
              Type: map<string,string>
              Comment: Custom key/value pairs.
            - Name: log_level
              Type: string
              Comment: Log level of the log event.
            - Name: log_original
              Type: string
              Comment: Original log message with light interpretation only (encoding, newlines).
            - Name: message
              Type: string
              Comment: Log message optimized for viewing in a log viewer.
            - Name: network_application
              Type: string
              Comment: Application level protocol name.
            - Name: network_bytes
              Type: bigint
              Comment: Total bytes transferred in both directions.
            - Name: network_community_id
              Type: string
              Comment: A hash of source and destination IPs and ports.
            - Name: network_direction
              Type: string
              Comment: Direction of the network traffic.
            - Name: network_forwarded_ip
              Type: string
              Comment: Host IP address when the source IP address is the proxy.
            - Name: network_iana_number
              Type: string
              Comment: IANA Protocol Number.
            - Name: network_name
              Type: string
              Comment: Name given by operators to sections of their network.
            - Name: network_packets
              Type: bigint
              Comment: Total packets transferred in both directions.
            - Name: network_protocol
              Type: string
              Comment: L7 Network protocol name.
            - Name: network_transport
              Type: string
              Comment: Protocol Name corresponding to the field `iana_number`.
            - Name: network_type
              Type: string
              Comment: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim,
                etc
            - Name: observer_geo_city_name
              Type: string
              Comment: City name.
            - Name: observer_geo_continent_name
              Type: string
              Comment: Name of the continent.
            - Name: observer_geo_country_iso_code
              Type: string
              Comment: Country ISO code.
            - Name: observer_geo_country_name
              Type: string
              Comment: Country name.
            - Name: observer_geo_location
              Type: struct<lat:double,lon:double>
              Comment: Longitude and latitude.
            - Name: observer_geo_name
              Type: string
              Comment: User-defined description of a location.
            - Name: observer_geo_region_iso_code
              Type: string
              Comment: Region ISO code.
            - Name: observer_geo_region_name
              Type: string
              Comment: Region name.
            - Name: observer_hostname
              Type: string
              Comment: Hostname of the observer.
            - Name: observer_ip
              Type: string
              Comment: IP address of the observer.
            - Name: observer_mac
              Type: string
              Comment: MAC address of the observer
            - Name: observer_os_family
              Type: string
              Comment: OS family (such as redhat, debian, freebsd, windows).
            - Name: observer_os_full
              Type: string
              Comment: Operating system name, including the version or code name.
            - Name: observer_os_kernel
              Type: string
              Comment: Operating system kernel version as a raw string.
            - Name: observer_os_name
              Type: string
              Comment: Operating system name, without the version.
            - Name: observer_os_platform
              Type: string
              Comment: Operating system platform (such centos, ubuntu, windows).
            - Name: observer_os_version
              Type: string
              Comment: Operating system version as a raw string.
            - Name: observer_serial_number
              Type: string
              Comment: Observer serial number.
            - Name: observer_type
              Type: string
              Comment: The type of the observer the data is coming from.
            - Name: observer_vendor
              Type: string
              Comment: observer vendor information.
            - Name: observer_version
              Type: string
              Comment: Observer version.
            - Name: organization_id
              Type: string
              Comment: Unique identifier for the organization.
            - Name: organization_name
              Type: string
              Comment: Organization name.
            - Name: os_family
              Type: string
              Comment: OS family (such as redhat, debian, freebsd, windows).
            - Name: os_full
              Type: string
              Comment: Operating system name, including the version or code name.
            - Name: os_kernel
              Type: string
              Comment: Operating system kernel version as a raw string.
            - Name: os_name
              Type: string
              Comment: Operating system name, without the version.
            - Name: os_platform
              Type: string
              Comment: Operating system platform (such centos, ubuntu, windows).
            - Name: os_version
              Type: string
              Comment: Operating system version as a raw string.
            - Name: process_args
              Type: string
              Comment: Array of process arguments.
            - Name: process_executable
              Type: string
              Comment: Absolute path to the process executable.
            - Name: process_hash_md5
              Type: string
              Comment: MD5 hash.
            - Name: process_hash_sha1
              Type: string
              Comment: SHA1 hash.
            - Name: process_hash_sha256
              Type: string
              Comment: SHA256 hash.
            - Name: process_hash_sha512
              Type: string
              Comment: SHA512 hash.
            - Name: process_name
              Type: string
              Comment: Process name.
            - Name: process_pgid
              Type: bigint
              Comment: Identifier of the group of processes the process belongs to.
            - Name: process_pid
              Type: bigint
              Comment: Process id.
            - Name: process_ppid
              Type: bigint
              Comment: Parent process' pid.
            - Name: process_start
              Type: timestamp
              Comment: The time the process started.
            - Name: process_thread_id
              Type: bigint
              Comment: Thread ID.
            - Name: process_title
              Type: string
              Comment: Process title.
            - Name: process_uptime
              Type: bigint
              Comment: Seconds the process has been up.
            - Name: process_working_directory
              Type: string
              Comment: The working directory of the process.
            - Name: related_ip
              Type: string
              Comment: All of the IPs seen on your event.
            - Name: server_address
              Type: string
              Comment: Server network address.
            - Name: server_bytes
              Type: bigint
              Comment: Bytes sent from the server to the client.
            - Name: server_domain
              Type: string
              Comment: Server domain.
            - Name: server_geo_city_name
              Type: string
              Comment: City name.
            - Name: server_geo_continent_name
              Type: string
              Comment: Name of the continent.
            - Name: server_geo_country_iso_code
              Type: string
              Comment: Country ISO code.
            - Name: server_geo_country_name
              Type: string
              Comment: Country name.
            - Name: server_geo_location
              Type: struct<lat:double,lon:double>
              Comment: Longitude and latitude.
            - Name: server_geo_name
              Type: string
              Comment: User-defined description of a location.
            - Name: server_geo_region_iso_code
              Type: string
              Comment: Region ISO code.
            - Name: server_geo_region_name
              Type: string
              Comment: Region name.
            - Name: server_ip
              Type: string
              Comment: IP address of the server.
            - Name: server_mac
              Type: string
              Comment: MAC address of the server.
            - Name: server_nat_ip
              Type: string
              Comment: Server NAT ip
            - Name: server_nat_port
              Type: bigint
              Comment: Server NAT port
            - Name: server_packets
              Type: bigint
              Comment: Packets sent from the server to the client.
            - Name: server_port
              Type: bigint
              Comment: Port of the server.
            - Name: server_user_domain
              Type: string
              Comment: Name of the directory the user is a member of.
            - Name: server_user_email
              Type: string
              Comment: User email address.
            - Name: server_user_full_name
              Type: string
              Comment: User's full name, if available.
            - Name: server_user_group_id
              Type: string
              Comment: Unique identifier for the group on the system/platform.
            - Name: server_user_group_name
              Type: string
              Comment: Name of the group.
            - Name: server_user_hash
              Type: string
              Comment: Unique user hash to correlate information for a user in anonymized form.
            - Name: server_user_id
              Type: string
              Comment: One or multiple unique identifiers of the user.
            - Name: server_user_name
              Type: string
              Comment: Short name or login of the user.
            - Name: service_ephemeral_id
              Type: string
              Comment: Ephemeral identifier of this service.
            - Name: service_id
              Type: string
              Comment: Unique identifier of the running service.
            - Name: service_name
              Type: string
              Comment: Name of the service.
            - Name: service_state
              Type: string
              Comment: Current state of the service.
            - Name: service_type
              Type: string
              Comment: The type of the service.
            - Name: service_version
              Type: string
              Comment: Version of the service.
            - Name: source_address
              Type: string
              Comment: Source network address.
            - Name: source_bytes
              Type: bigint
              Comment: Bytes sent from the source to the destination.
            - Name: source_domain
              Type: string
              Comment: Source domain.
            - Name: source_geo_city_name
              Type: string
              Comment: City name.
            - Name: source_geo_continent_name
              Type: string
              Comment: Name of the continent.
            - Name: source_geo_country_iso_code
              Type: string
              Comment: Country ISO code.
            - Name: source_geo_country_name
              Type: string
              Comment: Country name.
            - Name: source_geo_location
              Type: struct<lat:double,lon:double>
              Comment: Longitude and latitude.
            - Name: source_geo_name
              Type: string
              Comment: User-defined description of a location.
            - Name: source_geo_region_iso_code
              Type: string
              Comment: Region ISO code.
            - Name: source_geo_region_name
              Type: string
              Comment: Region name.
            - Name: source_ip
              Type: string
              Comment: IP address of the source.
            - Name: source_mac
              Type: string
              Comment: MAC address of the source.
            - Name: source_nat_ip
              Type: string
              Comment: Source NAT ip
            - Name: source_nat_port
              Type: bigint
              Comment: Source NAT port
            - Name: source_packets
              Type: bigint
              Comment: Packets sent from the source to the destination.
            - Name: source_port
              Type: bigint
              Comment: Port of the source.
            - Name: source_user_domain
              Type: string
              Comment: Name of the directory the user is a member of.
            - Name: source_user_email
              Type: string
              Comment: User email address.
            - Name: source_user_full_name
              Type: string
              Comment: User's full name, if available.
            - Name: source_user_group_id
              Type: string
              Comment: Unique identifier for the group on the system/platform.
            - Name: source_user_group_name
              Type: string
              Comment: Name of the group.
            - Name: source_user_hash
              Type: string
              Comment: Unique user hash to correlate information for a user in anonymized form.
            - Name: source_user_id
              Type: string
              Comment: One or multiple unique identifiers of the user.
            - Name: source_user_name
              Type: string
              Comment: Short name or login of the user.
            - Name: tags
              Type: string
              Comment: List of keywords used to tag each event.
            - Name: url_domain
              Type: string
              Comment: Domain of the url.
            - Name: url_fragment
              Type: string
              Comment: Portion of the url after the `#`.
            - Name: url_full
              Type: string
              Comment: Full unparsed URL.
            - Name: url_original
              Type: string
              Comment: Unmodified original url as seen in the event source.
            - Name: url_password
              Type: string
              Comment: Password of the request.
            - Name: url_path
              Type: string
              Comment: Path of the request, such as "/search".
            - Name: url_port
              Type: bigint
              Comment: Port of the request, such as 443.
            - Name: url_query
              Type: string
              Comment: Query string of the request.
            - Name: url_scheme
              Type: string
              Comment: Scheme of the url.
            - Name: url_username
              Type: string
              Comment: Username of the request.
            - Name: user_domain
              Type: string
              Comment: Name of the directory the user is a member of.
            - Name: user_email
              Type: string
              Comment: User email address.
            - Name: user_full_name
              Type: string
              Comment: User's full name, if available.
            - Name: user_group_id
              Type: string
              Comment: Unique identifier for the group on the system/platform.
            - Name: user_group_name
              Type: string
              Comment: Name of the group.
            - Name: user_hash
              Type: string
              Comment: Unique user hash to correlate information for a user in anonymized form.
            - Name: user_id
              Type: string
              Comment: One or multiple unique identifiers of the user.
            - Name: user_name
              Type: string
              Comment: Short name or login of the user.
            - Name: user_agent_device_name
              Type: string
              Comment: Name of the device.
            - Name: user_agent_name
              Type: string
              Comment: Name of the user agent.
            - Name: user_agent_original
              Type: string
              Comment: Unparsed version of the user_agent.
            - Name: user_agent_os_family
              Type: string
              Comment: OS family (such as redhat, debian, freebsd, windows).
            - Name: user_agent_os_full
              Type: string
              Comment: Operating system name, including the version or code name.
            - Name: user_agent_os_kernel
              Type: string
              Comment: Operating system kernel version as a raw string.
            - Name: user_agent_os_name
              Type: string
              Comment: Operating system name, without the version.
            - Name: user_agent_os_platform
              Type: string
              Comment: Operating system platform (such centos, ubuntu, windows).
            - Name: user_agent_os_version
              Type: string
              Comment: Operating system version as a raw string.
            - Name: user_agent_version
              Type: string
              Comment: Version of the user agent.

  ClientSideTelemetryDeliveryStream:
    Type: AWS::KinesisFirehose::DeliveryStream
    Properties:
      DeliveryStreamType: DirectPut
      ExtendedS3DestinationConfiguration:
        BucketARN:
          Fn::GetAtt:
            - ClientSideTelemetryBucket
            - Arn
        RoleARN:
          Fn::GetAtt:
            - ClientSideTelemetryDeliveryStreamRole
            - Arn

        # The next two properties tell Firehose where to write batched data
        Prefix: "data/ecs/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"
        ErrorOutputPrefix: "errors/ecs/!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}/"

        # Parquet queries are more efficient with larger files, so we'll instruct
        # Firehose to batch data for 15 minutes or until 128 MBs are waiting to be
        # processed.
        BufferingHints:
          IntervalInSeconds: 900
          SizeInMBs: 128

        # Snappy compression is intrinsic to the parquet format, so disable any
        # further compression.
        CompressionFormat: UNCOMPRESSED

        # This property tells Firehose that we'll be sending JSON and want it
        # converted to parquet
        DataFormatConversionConfiguration:
          Enabled: True
          InputFormatConfiguration:
            Deserializer:
              OpenXJsonSerDe:
                ColumnToJsonKeyMappings:
                  ts: "@timestamp"
                ConvertDotsInJsonKeysToUnderscores: True
          OutputFormatConfiguration:
            Serializer:
              ParquetSerDe: {}
          SchemaConfiguration:
            CatalogId:
              Ref: AWS::AccountId
            DatabaseName:
              Ref: ClientSideTelemetryDatabase
            Region:
              Ref: AWS::Region
            TableName:
              Ref: ClientSideEcsTelemetryTable
            VersionId: LATEST
            RoleARN:
              Fn::GetAtt:
                - ClientSideTelemetrySchemaReaderRole
                - Arn

  ClientSideTelemetryDeliveryStreamRole:
    Type: AWS::IAM::Role
    Properties:
      Path: "/"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          -
            Action:
              - 'sts:AssumeRole'
            Effect: Allow
            Principal:
              Service:
                - 'firehose.amazonaws.com'
      Policies:
        -
          PolicyName: 'ClientSideTelemetryDeliveryStreamPolicy'
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              -
                Effect: Allow
                Action:
                  - 's3:AbortMultipartUpload'
                  - 's3:GetBucketLocation'
                  - 's3:GetObject'
                  - 's3:ListBucket'
                  - 's3:ListBucketMultipartUploads'
                  - 's3:PutObject'
                Resource:
                  - Fn::Sub: "${ClientSideTelemetryBucket.Arn}"
                  - Fn::Sub: "${ClientSideTelemetryBucket.Arn}/*"

  ClientSideTelemetrySchemaReaderRole:
    Type: AWS::IAM::Role
    Properties:
      Path: "/"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          -
            Action:
              - 'sts:AssumeRole'
            Effect: Allow
            Principal:
              Service:
                - 'firehose.amazonaws.com'
      Policies:
        -
          PolicyName: 'ClientSideTelemetrySchemaReaderPolicy'
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              -
                Effect: Allow
                Action:
                  - 'glue:GetTableVersions'
                Resource:
                  - '*'

  ClientSideTelemetryLambdaRole:
    Type: AWS::IAM::Role
    Properties:
      Path: "/"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          -
            Action:
              - 'sts:AssumeRole'
            Effect: Allow
            Principal:
              Service:
                - 'lambda.amazonaws.com'
                - 'edgelambda.amazonaws.com'
      ManagedPolicyArns:
        - 'arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
      Policies:
        -
          PolicyName: 'ClientSideTelemetryLambdaPolicy'
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              -
                Effect: Allow
                Action:
                  - 'firehose:PutRecord'
                Resource:
                  Fn::GetAtt:
                    - ClientSideTelemetryDeliveryStream
                    - Arn
