Skip to main content
Sumo Logic

Collect Logs for Google App Engine

Before you can install the Google  App Engine App, you must configure log collection.

This page has instructions for configuring Google App Engine to send logs to Sumo.

Configure App Engine to export logs to Stackdriver

If you haven't already done so, set up Google App Engine to export logs to Stackdriver. For more information, see Overview of Logs Export in GCP.

Set up a Google Cloud Platform source and Pub/Sub topic

In this step, you set up an Google Cloud Platform source in Sumo, register it with Google, and create a Pub/Sub topic to send data to the source source. Follow the instructions in Google Cloud Platform Source.

Create export of App Engine logs from Stackdriver

In this step you export App Engine logs from Stackdriver to the Pub/Sub topic you created in the previous step.

  1. Click Logging in the STACKDRIVER section in the left hand pane of the GCP console.
    gcp6.png
  2. Go to Exports. Click Create Export.
    gcp7.png
  3. In the center pane, select "GAE Application" as the service to filter the logs. 
    gcp8.png

  4. In the Edit Export pane on the right:

    1. Set the Sink Name. For example, "gce-applications".
    2. Set Sink Service to “Cloud Pub/Sub”.
    3. Set Sink Destination to your Pub/Sub topic. For example, "pub-sub-logs".
    4. Click Create Sink.

Sample Log Message

{
  "message": {
    "data": {
      "httpRequest": {
        "status": 200
      },
      "insertId": "5a0e593e000d3494692375e0",
      "labels": {
        "clone_id": "00c61b117c27a4ec1f60970fdd9db838ca7ce717bfea65f950122a708e22eff25699065ba561"
      },
      "logName": "projects/bmlabs-loggen/logs/appengine.googleapis.com%2Frequest_log",
      "operation": {
        "first": true,
        "id": "5a0e593e00ff07071f75c6d2240001737e776b2d6465760001626967736b7966343a363932382d30333566343662000100",
        "last": true,
        "producer": "appengine.googleapis.com/request_id"
      },
      "protoPayload": {
        "@type": "type.googleapis.com/google.appengine.logging.v1.RequestLog",
        "appEngineRelease": "1.9.54",
        "appId": "s~bmlabs-loggen",
        "cost": 9.957699999999999e-8,
        "endTime": "2018-01-26T11:35:11.374UTC",
        "finished": true,
        "first": true,
        "host": "bigskyf4.bmlabs-loggen.appspot.com",
        "httpVersion": "HTTP/1.1",
        "instanceId": "00c61b117c27a4ec1f60970fdd9db838ca7ce717bfea65f950122a708e22eff25699065ba561",
        "instanceIndex": -1,
        "ip": "0.1.0.2",
        "latency": "0.404415s",
        "line": [
          {
            "logMessage": "Setting in-memory RuntimeSettings:6928-035f46b.",
            "severity": "DEBUG",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/base/runtime_settings.py",
              "functionName": "_get_data",
              "line": "180"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "TASK-INFO: {\"gae_latency_seconds\": 0.16422700881958008, \"task_eta\": 1510889790.3144629, \"execution_count\": 0, \"ran\": 1510889790.4786899, \"expected_eta\": 1510889790.3136301, \"retry_count\": 0, \"latency_seconds\": 0.16505980491638184}",
            "severity": "DEBUG",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/base/deferred/deferred.py",
              "functionName": "_log_task_info",
              "line": "752"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "Deferred 8a3a365c-4ed3-40ae-8a1c-faff06a9464c running function audit.actions._write with args = () and kwargs = {'account': <authmodel.auth.Account object at 0xf46cae50>, 'entity_keys': [datastore_types.Key.from_path(u'Membership', 4674727938359296L, _app=u's~bmlabs-loggen')], 'timestamp': datetime.datetime(2017, 11, 17, 3, 36, 30, 307040), 'audit_type': 'account_deactivated', 'user': <authmodel.auth.WFUser object at 0xf41fd730>, 'message': u'Service Docs exited the service_docs account.', 'ip_address': '52.4.205.181'}.",
            "severity": "INFO",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/base/deferred/deferred.py",
              "functionName": "_run_function",
              "line": "574"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "TASK-DONE: {\"queue_name\": \"audit-log\", \"worker_id\": \"8a3a365c-4ed3-40ae-8a1c-faff06a9464c\", \"task_name\": \"27563511594509108661\"}",
            "severity": "INFO",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/base/deferred/deferred.py",
              "functionName": "_run_deferred",
              "line": "921"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "gstats: ['event'] messages are disabled by default.",
            "severity": "DEBUG",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/external_libs/gstats/client.py",
              "functionName": "__init__",
              "line": "424"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "gstats: statsd sent 16 metric keys with 18 metric values in 2 udp packets in [17, 15] ms",
            "severity": "DEBUG",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/external_libs/gstats/listeners/statsd.py",
              "functionName": "send",
              "line": "362"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "GSTATSDATA: eJytlU1v2zAMhv9KofPqWHJkOQF6G7Dr2u60oRAEm0uM+iOTqLRBkP8+Sh3W2d3QOPPNskTx4cvX9JFh3YJ2YGtwbH11ZHZXauzRNLrsfYfx3d40Pm5/O8bzDk27oyUTKVfXnF9z9SXN1lm+ztKk4ELl6Vf24eoljs6p08OJlmjc4w8PHhILaA+XJ5CKDxOkpwdatYCmMmjilTuDW3rofNOcQnJ4BiqM7o67vqsx5GOt+33PhAKFGubPlvItQSz1T4Sw47C3oPeZ/uxRV94arPtuBqSlSIdIXL2DRHtjjV4bFNTyAe3yJuViepNaaEtTbkF/glnVyfhyyCIo23mRMhvpem6kHOec3o+BYe58d+uBPps5XSNHrhF/adLZptkY0I1B6MoDTZSy7yo3ZLwEMc/FyEYJQReF4lLKTBRZvuITjHU/s7EKNTbWfzX5IzSAMGuLi5F+fDUNMfb43TxSqF9KlE0NcWKwDYWgu7lJk1WiIgJY91IVC+9S9i8OZnxVo276DXvFYQtttot4ZFHBd7AWqkU8mJgyqOUS/WRrhBCz7V1kcPrpUVewZ7GMwy7ejqRyG35BkTcMOAKLAIHYOxp6FSQ0US+Ze3z89yN7koo/AQ4fOc0=",
            "severity": "DEBUG",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/external_libs/gstats/listeners/log.py",
              "functionName": "send",
              "line": "190"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "gstats: .send() took 36 ms",
            "severity": "DEBUG",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/external_libs/gstats/client.py",
              "functionName": "send",
              "line": "678"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "PROFILE: eJyt0U9rwjAYBvCvUnL2YFpj586C180eZZSQPmKxaSB5WxTpd1+6DdY2sFXx9v6BXx7e3BguUDmVGuw1Sl7Wi4iZFvYEWfjB0rdKVpXz9eHGisZKKk3tu/hn40u2A2VXR9AZSXLMbxxsW6qeZO5r0w/N8ehAvdotoqdhyT/YRNDQSqoTRkY6Nfj9Bv8zSDYTEUGSVAyYt2bKFJKkI2ORt8mI2sRBnuVA2jf1ewN7ncvFafBrfDPwtqhAmK2F6cT9J0/Cm3MxOnpdTBwLbQi5M+r8/cYvJlYBtnoYS3n30X0CIfoHPA==",
            "severity": "INFO",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/external_libs/appstats_logger/middleware.py",
              "functionName": "_stop_recording",
              "line": "119"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          },
          {
            "logMessage": "appstats: http://bigskyf4.bmlabs-loggen.appspot.com/stats/details?time=1510889790466",
            "severity": "INFO",
            "sourceLocation": {
              "file": "/base/data/home/apps/s~bmlabs-loggen/bigskyf4:6928-035f46b.405569666796645761/lib/appstats/recording.py",
              "functionName": "save",
              "line": "739"
            },
            "time": "2018-01-26T11:35:11.374UTC"
          }
        ],
        "megaCycles": "196",
        "method": "POST",
        "moduleId": "bigskyf4",
        "referrer": "https://bmlabs-loggen.wdesk.org/auth/logout/",
        "requestId": "5a0e593e00ff07071f75c6d2240001737e776b2d6465760001626967736b7966343a363932382d30333566343662000100",
        "resource": "/_ah/queue/deferred/audit.actions._write",
        "responseSize": "117",
        "startTime": "2018-01-26T11:35:11.374UTC",
        "status": 200,
        "taskName": "27563511594509108661",
        "taskQueueName": "audit-log",
        "traceId": "a736a2573bd0d51cce823884cc2b3460",
        "urlMapEntry": "main.app",
        "userAgent": "AppEngine-Google; (+http://code.google.com/appengine)",
        "versionId": "6928-035f46b"
      },
      "receiveTimestamp": "2018-01-26T11:35:11.374UTC",
      "resource": {
        "labels": {
          "module_id": "bigskyf4",
          "project_id": "bmlabs-loggen",
          "version_id": "6928-035f46b",
          "zone": "us14"
        },
        "type": "gae_app"
      },
      "severity": "INFO",
      "timestamp": "2018-01-26T11:35:11.374UTC"
    },
    "attributes": {
      "logging.googleapis.com/timestamp": "2018-01-26T11:35:11.374UTC"
    },
    "message_id": "172694189887110",
    "messageId": "172694189887110",
    "publish_time": "2018-01-26T11:35:11.374UTC",
    "publishTime": "2018-01-26T11:35:11.374UTC"
  },
  "subscription": "projects/bmlabs-loggen/subscriptions/sumo-test"
}

Query sample

Status codes over time

_collector="HTTP Source for GCP Pub/Sub" logName resource timestamp
| json "message.data.resource.type" as type
| parse regex "\"logName\":\"(?<log_name>[^\"]+)\"" 
| where type = "gae_app" | where log_name matches "projects/*/logs/appengine.googleapis.com%2Frequest_log"
| json "message.data.resource.labels" as labels
| json field=labels "module_id", "version_id", "zone", "project_id" as service, version, zone, project
| json "message.data.protoPayload.appId", "message.data.protoPayload.status" as app_id, status_code
| timeslice 30m
| if(status_code matches "20*", 1, 0) as resp_200
| if(status_code matches "30*", 1, 0) as resp_300
| if(status_code matches "40*", 1, 0) as resp_400
| if(status_code matches "50*", 1, 0) as resp_500
| if(!(status_code matches "20*" or status_code matches "30*" or status_code matches "40*" or status_code matches "50*"), 1, 0) as resp_others
| sum(resp_200) as tot_200, sum(resp_300) as tot_300, sum(resp_400) as tot_400, sum(resp_500) as tot_500, sum(resp_others) as tot_others by _timeslice