We're updating the issue view to help you get more done. 

Kinesis search logging failed

Description

Here is the error from the logs:

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 2019-06-08T07:59:00,438 ERROR [duler_Worker-13] [org.sagebionetworks.workers.util.semaphore.SemaphoreGatedRunnerImpl] - Error on key searchIndexWorker: com.amazonaws.services.kinesisfirehose.model.AmazonKinesisFirehoseException: User: arn:aws:sts::325565585839:assumed-role/prod-268-shared-resources-prod268SynapesRepoWorker-1WNULTQ1EDP05/i-04a4ccddd3a6c3294 is not authorized to perform: firehose:PutRecordBatch on resource: arn:aws:firehose:us-east-1:325565585839:deliverystream/prod268cloudsearchDocumentGeneration (Service: AmazonKinesisFirehose; Status Code: 400; Error Code: AccessDeniedException; Request ID: fb4eed43-eb49-f583-ae3b-8047f4c5a615) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1712) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1367) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1113) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:770) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:744) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:726) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:686) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:668) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:532) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:512) ~[aws-java-sdk-core-1.11.535.jar:?] at com.amazonaws.services.kinesisfirehose.AmazonKinesisFirehoseClient.doInvoke(AmazonKinesisFirehoseClient.java:1350) ~[aws-java-sdk-kinesis-1.11.535.jar:?] at com.amazonaws.services.kinesisfirehose.AmazonKinesisFirehoseClient.invoke(AmazonKinesisFirehoseClient.java:1317) ~[aws-java-sdk-kinesis-1.11.535.jar:?] at com.amazonaws.services.kinesisfirehose.AmazonKinesisFirehoseClient.invoke(AmazonKinesisFirehoseClient.java:1306) ~[aws-java-sdk-kinesis-1.11.535.jar:?] at com.amazonaws.services.kinesisfirehose.AmazonKinesisFirehoseClient.executePutRecordBatch(AmazonKinesisFirehoseClient.java:883) ~[aws-java-sdk-kinesis-1.11.535.jar:?] at com.amazonaws.services.kinesisfirehose.AmazonKinesisFirehoseClient.putRecordBatch(AmazonKinesisFirehoseClient.java:855) ~[aws-java-sdk-kinesis-1.11.535.jar:?] at org.sagebionetworks.kinesis.AwsKinesisFirehoseLoggerImpl.logBatch(AwsKinesisFirehoseLoggerImpl.java:22) ~[lib-logging-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.kinesis.AwsKinesisFirehoseLoggerImpl$$FastClassBySpringCGLIB$$bcf94a7.invoke(<generated>) ~[lib-logging-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:749) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.profiler.Profiler.doBasicProfiling(Profiler.java:37) ~[lib-logging-268.02-13-g9474d34.jar:268.02-13-g9474d34] at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_191] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_191] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:688) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.kinesis.AwsKinesisFirehoseLoggerImpl$$EnhancerBySpringCGLIB$$bc2ddbf7.logBatch(<generated>) ~[lib-logging-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.search.CloudsSearchDomainClientAdapter.updateAndSendKinesisLogRecords(CloudsSearchDomainClientAdapter.java:103) ~[lib-search-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.search.CloudsSearchDomainClientAdapter.sendDocuments(CloudsSearchDomainClientAdapter.java:71) ~[lib-search-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.search.SearchDaoImpl.sendDocuments(SearchDaoImpl.java:75) ~[lib-search-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.search.SearchDaoImpl$$FastClassBySpringCGLIB$$fc8f06db.invoke(<generated>) ~[lib-search-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:749) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.profiler.Profiler.doBasicProfiling(Profiler.java:37) ~[lib-logging-268.02-13-g9474d34.jar:268.02-13-g9474d34] at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_191] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_191] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:688) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.search.SearchDaoImpl$$EnhancerBySpringCGLIB$$bc3b3313.sendDocuments(<generated>) ~[lib-search-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.repo.manager.search.SearchManagerImpl.documentChangeMessages(SearchManagerImpl.java:120) ~[repository-managers-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.repo.manager.search.SearchManagerImpl$$FastClassBySpringCGLIB$$44a48205.invoke(<generated>) ~[repository-managers-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:749) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.profiler.Profiler.doBasicProfiling(Profiler.java:37) ~[lib-logging-268.02-13-g9474d34.jar:268.02-13-g9474d34] at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_191] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_191] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:688) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.repo.manager.search.SearchManagerImpl$$EnhancerBySpringCGLIB$$bbc4eed1.documentChangeMessages(<generated>) ~[repository-managers-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.search.workers.sqs.search.SearchQueueWorker.run(SearchQueueWorker.java:40) ~[classes/:268.02-13-g9474d34] at org.sagebionetworks.search.workers.sqs.search.SearchQueueWorker$$FastClassBySpringCGLIB$$eadefac2.invoke(<generated>) ~[classes/:268.02-13-g9474d34] at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218) ~[spring-core-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:749) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.worker.utils.WorkerProfiler.profileMessageDrivenRunner(WorkerProfiler.java:50) ~[classes/:268.02-13-g9474d34] at sun.reflect.GeneratedMethodAccessor80.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_191] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_191] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.MethodInvocationProceedingJoinPoint.proceed(MethodInvocationProceedingJoinPoint.java:88) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.profiler.Profiler.doBasicProfiling(Profiler.java:37) ~[lib-logging-268.02-13-g9474d34.jar:268.02-13-g9474d34] at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_191] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_191] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethodWithGivenArgs(AbstractAspectJAdvice.java:644) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AbstractAspectJAdvice.invokeAdviceMethod(AbstractAspectJAdvice.java:633) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.aspectj.AspectJAroundAdvice.invoke(AspectJAroundAdvice.java:70) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:93) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:688) ~[spring-aop-5.1.5.RELEASE.jar:5.1.5.RELEASE] at org.sagebionetworks.search.workers.sqs.search.SearchQueueWorker$$EnhancerBySpringCGLIB$$b02acaa2.run(<generated>) ~[classes/:268.02-13-g9474d34] at org.sagebionetworks.asynchronous.workers.changes.ChangeMessageBatchProcessor.runAsBatch(ChangeMessageBatchProcessor.java:64) ~[lib-worker-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.asynchronous.workers.changes.ChangeMessageBatchProcessor.run(ChangeMessageBatchProcessor.java:50) ~[lib-worker-268.02-13-g9474d34.jar:268.02-13-g9474d34] at org.sagebionetworks.workers.util.aws.message.PollingMessageReceiverImpl.processMessage(PollingMessageReceiverImpl.java:189) ~[worker-utilities-1.1.1.jar:?] at org.sagebionetworks.workers.util.aws.message.PollingMessageReceiverImpl.run(PollingMessageReceiverImpl.java:121) ~[worker-utilities-1.1.1.jar:?] at org.sagebionetworks.common.util.progress.AutoProgressingRunner$1.call(AutoProgressingRunner.java:21) ~[common-utilities-1.0.10.jar:?] at org.sagebionetworks.common.util.progress.AutoProgressingRunner$1.call(AutoProgressingRunner.java:17) ~[common-utilities-1.0.10.jar:?] at org.sagebionetworks.common.util.progress.AutoProgressingCallable$1.call(AutoProgressingCallable.java:66) ~[common-utilities-1.0.10.jar:?] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_191] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_191] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_191] at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_191]

Environment

None

Status

Assignee

Ziming Dong

Reporter

John Hill

Labels

None

Validator

John Hill

Release Version History

None

Fix versions

Priority

Critical