LOGS_MODEL: splunk
LOGS_MODEL_CONFIG: 
    producer: splunk
    splunk_config:
        host: xx.xx.xx.xx
        port: 8090
        bearer_token: xxxxx
        url_scheme: https
        verify_ssl: False
        index_prefix: test

2024-02-26T07:59:53.645936693Z gunicorn-registry stdout | 2024-02-26 07:59:53,643 [268] [ERROR] [data.logs_model.logs_producer.splunk_logs_producer] SplunkLogsProducer exception sending log to Splunk: HTTP 403 Forbidden -- insufficient permission to access this resource2024-02-26T07:59:53.645936693Z gunicorn-registry stdout | Traceback (most recent call last):2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |   File "/quay-registry/data/logs_model/logs_producer/splunk_logs_producer.py", line 67, in send2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |     self.index.submit(log, sourcetype="access_combined", host="quay")2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/client.py", line 2229, in submit2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |     self.service.post(PATH_RECEIVERS_SIMPLE, body=event, **args)2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 320, in wrapper2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |     return request_fun(self, *args, **kwargs)2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 79, in new_f2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |     val = f(*args, **kwargs)2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 806, in post2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |     response = self.http.post(path, all_headers, **query)2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 1297, in post2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |     return self.request(url, message)2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 1326, in request2024-02-26T07:59:53.645936693Z gunicorn-registry stdout |     raise HTTPError(response)2024-02-26T07:59:53.645936693Z gunicorn-registry stdout | splunklib.binding.HTTPError: HTTP 403 Forbidden -- insufficient permission to access this resource2024-02-26T07:59:53.646234410Z gunicorn-registry stdout | 2024-02-26 07:59:53,645 [268] [ERROR] [data.logs_model.splunk_logs_model] log_action failed2024-02-26T07:59:53.646234410Z gunicorn-registry stdout | Traceback (most recent call last):2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/quay-registry/data/logs_model/logs_producer/splunk_logs_producer.py", line 67, in send2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     self.index.submit(log, sourcetype="access_combined", host="quay")2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/client.py", line 2229, in submit2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     self.service.post(PATH_RECEIVERS_SIMPLE, body=event, **args)2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 320, in wrapper2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     return request_fun(self, *args, **kwargs)2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 79, in new_f2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     val = f(*args, **kwargs)2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 806, in post2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     response = self.http.post(path, all_headers, **query)2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 1297, in post2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     return self.request(url, message)2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 1326, in request2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     raise HTTPError(response)2024-02-26T07:59:53.646234410Z gunicorn-registry stdout | splunklib.binding.HTTPError: HTTP 403 Forbidden -- insufficient permission to access this resource2024-02-26T07:59:53.646234410Z gunicorn-registry stdout | During handling of the above exception, another exception occurred:2024-02-26T07:59:53.646234410Z gunicorn-registry stdout | Traceback (most recent call last):2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/quay-registry/data/logs_model/splunk_logs_model.py", line 86, in log_action2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     self._logs_producer.send(json.dumps(log_data, sort_keys=True, default=str))2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |   File "/quay-registry/data/logs_model/logs_producer/splunk_logs_producer.py", line 70, in send2024-02-26T07:59:53.646234410Z gunicorn-registry stdout |     raise LogSendException("SplunkLogsProducer exception sending log to Splunk: %s" % e)2024-02-26T07:59:53.646234410Z gunicorn-registry stdout | data.logs_model.logs_producer.LogSendException: SplunkLogsProducer exception sending log to Splunk: HTTP 403 Forbidden -- insufficient permission to access this resource2024-02-26T07:59:53.648376251Z gunicorn-registry stdout | 2024-02-26 07:59:53,646 [268] [ERROR] [gunicorn.error] Error handling request /v2/tlc3/gtin-srv-nuc-jee-incidentcreation-5/manifests/5.37.0-SNAPSHOT2024-02-26T07:59:53.648376251Z gunicorn-registry stdout | Traceback (most recent call last):2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/data/logs_model/logs_producer/splunk_logs_producer.py", line 67, in send2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     self.index.submit(log, sourcetype="access_combined", host="quay")2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/client.py", line 2229, in submit2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     self.service.post(PATH_RECEIVERS_SIMPLE, body=event, **args)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 320, in wrapper2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return request_fun(self, *args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 79, in new_f2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     val = f(*args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 806, in post2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     response = self.http.post(path, all_headers, **query)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 1297, in post2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return self.request(url, message)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/splunklib/binding.py", line 1326, in request2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     raise HTTPError(response)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout | splunklib.binding.HTTPError: HTTP 403 Forbidden -- insufficient permission to access this resource2024-02-26T07:59:53.648376251Z gunicorn-registry stdout | During handling of the above exception, another exception occurred:2024-02-26T07:59:53.648376251Z gunicorn-registry stdout | Traceback (most recent call last):2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/gunicorn/workers/base_async.py", line 55, in handle2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     self.handle_request(listener_name, req, client, addr)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/gunicorn/workers/ggevent.py", line 128, in handle_request2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     super().handle_request(listener_name, req, sock, addr)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/gunicorn/workers/base_async.py", line 108, in handle_request2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     respiter = self.wsgi(environ, resp.start_response)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/flask/app.py", line 2213, in __call__2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return self.wsgi_app(environ, start_response)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/werkzeug/middleware/proxy_fix.py", line 182, in __call__2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return self.app(environ, start_response)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/flask/app.py", line 2193, in wsgi_app2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     response = self.handle_exception(e)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/flask/app.py", line 2190, in wsgi_app2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     response = self.full_dispatch_request()2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/flask/app.py", line 1486, in full_dispatch_request2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     rv = self.handle_user_exception(e)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/flask/app.py", line 1484, in full_dispatch_request2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     rv = self.dispatch_request()2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/app/lib/python3.9/site-packages/flask/app.py", line 1469, in dispatch_request2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/decorators.py", line 228, in wrapper2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return func(*args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/decorators.py", line 91, in wrapper2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return func(*args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/v2/manifest.py", line 230, in wrapped2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return func(*args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/auth/registry_jwt_auth.py", line 175, in wrapper2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return func(*args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/v2/__init__.py", line 169, in wrapped2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return func(namespace_name, repo_name, *args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/decorators.py", line 164, in wrapper2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return func(*args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/decorators.py", line 189, in wrapper2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return func(*args, **kwargs)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/v2/manifest.py", line 262, in write_manifest_by_tagname2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     return _write_manifest_and_log(namespace_name, repo_name, manifest_ref, parsed)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/endpoints/v2/manifest.py", line 394, in _write_manifest_and_log2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     track_and_log("push_repo", repository_ref, tag=tag_name)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/util/audit.py", line 89, in track_and_log2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     logs_model.log_action(2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/data/logs_model/splunk_logs_model.py", line 86, in log_action2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     self._logs_producer.send(json.dumps(log_data, sort_keys=True, default=str))2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |   File "/quay-registry/data/logs_model/logs_producer/splunk_logs_producer.py", line 70, in send2024-02-26T07:59:53.648376251Z gunicorn-registry stdout |     raise LogSendException("SplunkLogsProducer exception sending log to Splunk: %s" % e)2024-02-26T07:59:53.648376251Z gunicorn-registry stdout | data.logs_model.logs_producer.LogSendException: SplunkLogsProducer exception sending log to Splunk: HTTP 403 Forbidden -- insufficient permission to access this resource2024-02-26T07:59:53.648736619Z gunicorn-registry stdout | 2024-02-26 07:59:53,647 [268] [INFO] [gunicorn.access]  - - [26/Feb/2024:07:59:53 +0000] "PUT /v2/xxx/xxxxx-5/manifests/5.37.0-SNAPSHOT HTTP/1.1" 500 0 "-" "-"2024-02-26T07:59:53.649353450Z nginx stdout | 10.131.2.5 (-) - - [26/Feb/2024:07:59:53 +0000] "PUT /v2/tlc3/gtin-srv-nuc-jee-incidentcreation-5/manifests/5.37.0-SNAPSHOT HTTP/1.1" 500 141 "-" "containers/5.22.0 (github.com/containers/image)" (0.393 2625 0.391)2024-02-26T07:59:54.904115985Z gcworker stdout | 2024-02-26 07:59:54,903 [70] [WARNING] [data.model.gc] Exception attempting to delete manifest sha256:5ecb080be46b642dd4c133e99e877bdf37c453dbd14109e304399ee8bd2bb28e from secscan service