Log Aggregation & APM
updated 23:01:41

Logs

Full-text and structured search across every ingested source. Without search, logs are just stale.

Volume
{service=~".+"}
300 lines
23:01:40.110infoapiPOST /checkout -> 200
23:01:40.108infopaymentsPOST /charge -> 200
23:01:40.107infoworkerPOST /process -> 200
23:01:40.022infopaymentsPOST /charge -> 200
23:01:40.022infoapiPOST /checkout -> 200
23:01:40.021infoworkerPOST /process -> 200
23:01:39.775infoworkerGET / -> 200
23:01:39.743infoworkerGET / -> 200
23:01:39.722infoworkerGET / -> 200
23:01:39.718infoworkerGET / -> 200
23:01:39.717infoworkerGET / -> 200
23:01:39.507infopaymentsGET / -> 200
23:01:39.505infoworkerPOST /process -> 200
23:01:39.394infopaymentsGET / -> 200
23:01:39.393infoworkerPOST /process -> 200
23:01:39.305errorapiPOST /checkout -> 500
23:01:39.284infoapiPOST /checkout -> 200
23:01:39.283infopaymentsPOST /charge -> 200
23:01:39.282infoworkerPOST /process -> 200
23:01:39.235infoapiPOST /checkout -> 200
23:01:39.234infopaymentsPOST /charge -> 200
23:01:39.233infoworkerPOST /process -> 200
23:01:39.220infoworkerGET / -> 200
23:01:39.214infoworkerGET / -> 200
23:01:39.200infopaymentsPOST /charge -> 200
23:01:39.199infoworkerPOST /process -> 200
23:01:39.194infoapiPOST /checkout -> 200
23:01:39.192infopaymentsPOST /charge -> 200
23:01:39.191infoworkerPOST /process -> 200
23:01:39.122infoworkerGET / -> 200
23:01:39.119infoworkerGET / -> 200
23:01:39.118infoworkerGET / -> 200
23:01:39.012infopaymentsGET / -> 200
23:01:39.011infoworkerPOST /process -> 200
23:01:39.001infoclickhouse (version 24.8.14.39 (official build))
23:01:39.001infoclickhouse10. ? @ 0x000075475e85c353
23:01:39.001infoclickhouse9. ? @ 0x000075475e937609
23:01:39.001infoclickhouse8. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:39.001infoclickhouse7. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::AsynchronousMetrics::start()::$_0>(DB::AsynchronousMetrics::start()::$_0&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d2c4a4e
23:01:39.001infoclickhouse6. DB::AsynchronousMetrics::update(std::chrono::time_point<std::chrono::system_clock, std::chrono::duration<long long, std::ratio<1l, 1000000l>>>, bool) @ 0x000000000d2b0bdb
23:01:39.001infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:39.001infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:39.001infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:39.001infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:39.001infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:39.001infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:39.001infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:39.001infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:39.001infoclickhouseCannot add message to the log: MemoryTracking: was 442.28 MiB, peak 4.50 GiB, free memory in arenas 26.74 MiB, will set to 448.96 MiB (RSS), difference: 6.68 MiB
23:01:38.948infoworkerPOST /process -> 200
23:01:38.948infopaymentsGET / -> 200
23:01:38.895infopaymentsGET / -> 200
23:01:38.894infoworkerPOST /process -> 200
23:01:38.857infoclickhouse (version 24.8.14.39 (official build))
23:01:38.857infoclickhouse15. ? @ 0x000075475e85c353
23:01:38.857infoclickhouse14. ? @ 0x000075475e937609
23:01:38.857infoclickhouse13. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:38.857infoclickhouse12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d254011
23:01:38.857infoclickhouse11. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false, true>, void*>) @ 0x000000000d24fec9
23:01:38.857infoclickhouse10. DB::MergeTreeBackgroundExecutor<DB::RoundRobinRuntimeQueue>::threadFunction() @ 0x0000000011f18eba
23:01:38.857infoclickhouse9. DB::ExecutableLambdaAdapter::executeStep() @ 0x0000000011ba2552
23:01:38.857infoclickhouse8. bool std::__function::__policy_invoker<bool ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageMergeTree::scheduleDataProcessingJob(DB::BackgroundJobsAssignee&)::$_1, bool ()>>(std::__function::__policy_storage const*) @ 0x00000000122d9595
23:01:38.857infoclickhouse7. DB::MergeTreeData::clearOldPartsFromFilesystem(bool) @ 0x0000000011f4ec10
23:01:38.857infoclickhouse6. DB::MergeTreeData::removePartsFinally(std::vector<std::shared_ptr<DB::IMergeTreeDataPart const>, std::allocator<std::shared_ptr<DB::IMergeTreeDataPart const>>> const&) @ 0x0000000011f4d2f7
23:01:38.857infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:38.857infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:38.857infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:38.857infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:38.856infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:38.856infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:38.856infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:38.856infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:38.856infoclickhouseCannot add message to the log: Removing 6 parts from memory: Parts: [202608_316633_316883_50, 202608_316884_316884_0, 202608_316885_316885_0, 202608_316886_316886_0, 202608_316887_316887_0, 202608_316888_316888_0]
23:01:38.853infoclickhouse (version 24.8.14.39 (official build))
23:01:38.853infoclickhouse17. ? @ 0x000075475e85c353
23:01:38.853infoclickhouse16. ? @ 0x000075475e937609
23:01:38.853infoclickhouse15. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:38.853infoclickhouse14. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d254011
23:01:38.853infoclickhouse13. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false, true>, void*>) @ 0x000000000d24fec9
23:01:38.853infoclickhouse12. DB::MergeTreeBackgroundExecutor<DB::RoundRobinRuntimeQueue>::threadFunction() @ 0x0000000011f18eba
23:01:38.853infoclickhouse11. DB::ExecutableLambdaAdapter::executeStep() @ 0x0000000011ba2552
23:01:38.853infoclickhouse10. bool std::__function::__policy_invoker<bool ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageMergeTree::scheduleDataProcessingJob(DB::BackgroundJobsAssignee&)::$_1, bool ()>>(std::__function::__policy_storage const*) @ 0x00000000122d9595
23:01:38.853infoclickhouse9. DB::MergeTreeData::clearOldPartsFromFilesystem(bool) @ 0x0000000011f4ec03
23:01:38.853infoclickhouse8. DB::MergeTreeData::clearPartsFromFilesystem(std::vector<std::shared_ptr<DB::IMergeTreeDataPart const>, std::allocator<std::shared_ptr<DB::IMergeTreeDataPart const>>> const&, bool, std::unordered_set<String, std::hash<String>, std::equal_to<String>, std::allocator<String>>*) @ 0x0000000011f4ee40
23:01:38.853infoclickhouse7. DB::MergeTreeData::clearPartsFromFilesystemImpl(std::vector<std::shared_ptr<DB::IMergeTreeDataPart const>, std::allocator<std::shared_ptr<DB::IMergeTreeDataPart const>>> const&, std::unordered_set<String, std::hash<String>, std::equal_to<String>, std::allocator<String>>*) @ 0x0000000011f4fe32
23:01:38.853infoclickhouse6. DB::MergeTreeData::clearPartsFromFilesystemImpl(std::vector<std::shared_ptr<DB::IMergeTreeDataPart const>, std::allocator<std::shared_ptr<DB::IMergeTreeDataPart const>>> const&, std::unordered_set<String, std::hash<String>, std::equal_to<String>, std::allocator<String>>*)::$_3::operator()() const @ 0x0000000011f53436
23:01:38.853infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:38.853infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:38.853infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:38.853infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:38.853infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:38.853infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:38.853infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:38.853infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:38.853infoclickhouseCannot add message to the log: Removing 6 parts from filesystem (serially): Parts: [202608_316633_316883_50, 202608_316884_316884_0, 202608_316885_316885_0, 202608_316886_316886_0, 202608_316887_316887_0, 202608_316888_316888_0]
23:01:38.853infoclickhouse (version 24.8.14.39 (official build))
23:01:38.853infoclickhouse15. ? @ 0x000075475e85c353
23:01:38.853infoclickhouse14. ? @ 0x000075475e937609
23:01:38.853infoclickhouse13. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:38.853infoclickhouse12. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<false, true>::ThreadFromGlobalPoolImpl<void ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>(void&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d254011
23:01:38.853infoclickhouse11. ThreadPoolImpl<ThreadFromGlobalPoolImpl<false, true>>::worker(std::__list_iterator<ThreadFromGlobalPoolImpl<false, true>, void*>) @ 0x000000000d24fec9
23:01:38.853infoclickhouse10. DB::MergeTreeBackgroundExecutor<DB::RoundRobinRuntimeQueue>::threadFunction() @ 0x0000000011f18eba
23:01:38.853infoclickhouse9. DB::ExecutableLambdaAdapter::executeStep() @ 0x0000000011ba2552
23:01:38.853infoclickhouse8. bool std::__function::__policy_invoker<bool ()>::__call_impl<std::__function::__default_alloc_func<DB::StorageMergeTree::scheduleDataProcessingJob(DB::BackgroundJobsAssignee&)::$_1, bool ()>>(std::__function::__policy_storage const*) @ 0x00000000122d9595
23:01:38.853infoclickhouse7. DB::MergeTreeData::clearOldPartsFromFilesystem(bool) @ 0x0000000011f4ebdf
23:01:38.853infoclickhouse6. DB::MergeTreeData::grabOldParts(bool) @ 0x0000000011f4b626
23:01:38.853infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:38.853infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:38.853infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:38.853infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:38.853infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:38.853infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:38.853infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:38.853infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:38.853infoclickhouseCannot add message to the log: Found 6 old parts to remove. Parts: [202608_316633_316883_50, 202608_316884_316884_0, 202608_316885_316885_0, 202608_316886_316886_0, 202608_316887_316887_0, 202608_316888_316888_0]
23:01:38.739infoworkerGET / -> 200
23:01:38.707infoworkerGET / -> 200
23:01:38.696infoworkerGET / -> 200
23:01:38.696infoworkerGET / -> 200
23:01:38.685infoapiPOST /checkout -> 200
23:01:38.683infoworkerPOST /process -> 200
23:01:38.683infoworkerGET / -> 200
23:01:38.683infopaymentsPOST /charge -> 200
23:01:38.667infoapiPOST /checkout -> 200
23:01:38.666warnpaymentsdownstream worker returned 500
23:01:38.666infopaymentsPOST /charge -> 200
23:01:38.665errorworkerPOST /process -> 500
23:01:38.573infocoolify-sentinel[GIN] 2026/08/09 - 23:01:38 | 200 | 35.968µs | 127.0.0.1 | GET "/api/health"
23:01:38.550errorpostgres-yckw0wkg8k4co0osg8kwco0k-1454499557732026-08-09 23:01:38.550 UTC [104356] FATAL: database "cid_user" does not exist
23:01:38.514infopaymentsPOST /charge -> 200
23:01:38.514infoapiGET / -> 200
23:01:38.513infoworkerPOST /process -> 200
23:01:38.446infoapiGET / -> 200
23:01:38.445infoworkerPOST /process -> 200
23:01:38.445infopaymentsPOST /charge -> 200
23:01:38.395infopaymentsGET / -> 200
23:01:38.394infoworkerPOST /process -> 200
23:01:38.306infopaymentsGET / -> 200
23:01:38.305infoworkerPOST /process -> 200
23:01:38.289infopaymentsGET / -> 200
23:01:38.288infoworkerPOST /process -> 200
23:01:38.116infopaymentsPOST /charge -> 200
23:01:38.116infoapiGET / -> 200
23:01:38.114infoworkerPOST /process -> 200
23:01:38.077infopaymentsPOST /charge -> 200
23:01:38.077infoapiPOST /checkout -> 200
23:01:38.076infoworkerPOST /process -> 200
23:01:38.070infoapiGET / -> 200
23:01:38.069infopaymentsPOST /charge -> 200
23:01:38.068infoworkerPOST /process -> 200
23:01:38.058infoapiPOST /checkout -> 200
23:01:38.057infopaymentsPOST /charge -> 200
23:01:38.056infoworkerPOST /process -> 200
23:01:37.940infoworkerGET / -> 200
23:01:37.921infoworkerGET / -> 200
23:01:37.872infoworkerGET / -> 200
23:01:37.853infoworkerGET / -> 200
23:01:37.839infoworkerGET / -> 200
23:01:37.673errorworkerPOST /process -> 500
23:01:37.673warnpaymentsdownstream worker returned 500
23:01:37.673infopaymentsGET / -> 200
23:01:37.658infopaymentsPOST /charge -> 200
23:01:37.658infoapiGET / -> 200
23:01:37.657infoworkerPOST /process -> 200
23:01:37.651infoclickhouse (version 24.8.14.39 (official build))
23:01:37.651infoclickhouse10. ? @ 0x000075475e85c353
23:01:37.651infoclickhouse9. ? @ 0x000075475e937609
23:01:37.651infoclickhouse8. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:37.651infoclickhouse7. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d2a2808
23:01:37.651infoclickhouse6. DB::SystemLog<DB::AsynchronousMetricLogElement>::savingThreadFunction() @ 0x00000000111a71ef
23:01:37.651infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:37.651infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:37.651infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:37.651infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:37.651infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:37.651infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:37.651infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:37.651infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:37.651infoclickhouseCannot add message to the log: Flushed system log up to offset 598488906
23:01:37.651infoclickhouse (version 24.8.14.39 (official build))
23:01:37.651infoclickhouse18. ? @ 0x000075475e85c353
23:01:37.651infoclickhouse17. ? @ 0x000075475e937609
23:01:37.651infoclickhouse16. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:37.651infoclickhouse15. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d2a2808
23:01:37.651infoclickhouse14. DB::SystemLog<DB::AsynchronousMetricLogElement>::savingThreadFunction() @ 0x00000000111a6cf3
23:01:37.651infoclickhouse13. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x00000000125c8368
23:01:37.651infoclickhouse12. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x00000000125c8950
23:01:37.651infoclickhouse11. DB::ExecutionThreadContext::executeTask() @ 0x00000000125d4087
23:01:37.651infoclickhouse10. DB::ExceptionKeepingTransform::work() @ 0x000000001283842e
23:01:37.651infoclickhouse9. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x0000000012838b2b
23:01:37.651infoclickhouse8. DB::MergeTreeSink::finishDelayedChunk() @ 0x00000000122e43ac
23:01:37.651infoclickhouse7. DB::MergeTreeData::renameTempPartAndAdd(std::shared_ptr<DB::IMergeTreeDataPart>&, DB::MergeTreeData::Transaction&, DB::DataPartsLock&) @ 0x0000000011f70725
23:01:37.651infoclickhouse6. DB::MergeTreeData::renameTempPartAndReplaceImpl(std::shared_ptr<DB::IMergeTreeDataPart>&, DB::MergeTreeData::Transaction&, DB::DataPartsLock&, std::vector<std::shared_ptr<DB::IMergeTreeDataPart const>, std::allocator<std::shared_ptr<DB::IMergeTreeDataPart const>>>*) @ 0x0000000011f6e004
23:01:37.651infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:37.651infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:37.651infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:37.651infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:37.651infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:37.651infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:37.651infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:37.651infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:37.651infoclickhouseCannot add message to the log: Renaming temporary part tmp_insert_202608_342793_342793_0 to 202608_342794_342794_0 with tid (1, 1, 00000000-0000-0000-0000-000000000000).
23:01:37.651infoclickhouse (version 24.8.14.39 (official build))
23:01:37.651infoclickhouse20. ? @ 0x000075475e85c353
23:01:37.651infoclickhouse19. ? @ 0x000075475e937609
23:01:37.651infoclickhouse18. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:37.651infoclickhouse17. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d2a2808
23:01:37.651infoclickhouse16. DB::SystemLog<DB::AsynchronousMetricLogElement>::savingThreadFunction() @ 0x00000000111a6cf3
23:01:37.651infoclickhouse15. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x00000000125c8368
23:01:37.651infoclickhouse14. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x00000000125c8950
23:01:37.651infoclickhouse13. DB::ExecutionThreadContext::executeTask() @ 0x00000000125d4087
23:01:37.651infoclickhouse12. DB::ExceptionKeepingTransform::work() @ 0x0000000012838355
23:01:37.651infoclickhouse11. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x0000000012838b2b
23:01:37.651infoclickhouse10. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x0000000012838e18
23:01:37.651infoclickhouse9. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x00000000129014b3
23:01:37.651infoclickhouse8. DB::MergeTreeSink::consume(DB::Chunk&) @ 0x00000000122e569c
23:01:37.651infoclickhouse7. DB::MergeTreeDataWriter::writeTempPartImpl(DB::BlockWithPartition&, std::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::shared_ptr<DB::Context const>, long, bool) @ 0x0000000012180714
23:01:37.651infoclickhouse6. DB::MergedBlockOutputStream::finalizePartAsync(std::shared_ptr<DB::IMergeTreeDataPart> const&, bool, DB::NamesAndTypesList const*, DB::MergeTreeDataPartChecksums*, std::vector<DB::ColumnWithTypeAndName, std::allocator<DB::ColumnWithTypeAndName>>*) @ 0x000000001213f95a
23:01:37.651infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:37.651infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:37.651infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:37.651infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:37.651infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:37.651infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:37.650infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:37.650infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:37.650infoclickhouseCannot add message to the log: filled checksums 202608_342793_342793_0 (state Temporary)
23:01:37.650infoclickhouse (version 24.8.14.39 (official build))
23:01:37.650infoclickhouse24. ? @ 0x000075475e85c353
23:01:37.650infoclickhouse23. ? @ 0x000075475e937609
23:01:37.650infoclickhouse22. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:37.650infoclickhouse21. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d2a2808
23:01:37.650infoclickhouse20. DB::SystemLog<DB::AsynchronousMetricLogElement>::savingThreadFunction() @ 0x00000000111a6cf3
23:01:37.650infoclickhouse19. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x00000000125c8368
23:01:37.650infoclickhouse18. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x00000000125c8950
23:01:37.650infoclickhouse17. DB::ExecutionThreadContext::executeTask() @ 0x00000000125d4087
23:01:37.650infoclickhouse16. DB::ExceptionKeepingTransform::work() @ 0x0000000012838355
23:01:37.650infoclickhouse15. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x0000000012838b2b
23:01:37.650infoclickhouse14. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x0000000012838e18
23:01:37.650infoclickhouse13. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x00000000129014b3
23:01:37.650infoclickhouse12. DB::MergeTreeSink::consume(DB::Chunk&) @ 0x00000000122e569c
23:01:37.650infoclickhouse11. DB::MergeTreeDataWriter::writeTempPartImpl(DB::BlockWithPartition&, std::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::shared_ptr<DB::Context const>, long, bool) @ 0x000000001217d863
23:01:37.650infoclickhouse10. DB::MergeTreeData::reserveSpacePreferringTTLRules(std::shared_ptr<DB::StorageInMemoryMetadata const> const&, unsigned long, DB::MergeTreeDataPartTTLInfos const&, long, unsigned long, bool, std::shared_ptr<DB::IDisk>) const @ 0x0000000011f9f859
23:01:37.650infoclickhouse9. DB::MergeTreeData::tryReserveSpacePreferringTTLRules(std::shared_ptr<DB::StorageInMemoryMetadata const> const&, unsigned long, DB::MergeTreeDataPartTTLInfos const&, long, unsigned long, bool, std::shared_ptr<DB::IDisk>) const @ 0x0000000011fa2060
23:01:37.650infoclickhouse8. DB::StoragePolicy::reserve(unsigned long, unsigned long) const @ 0x0000000010ba150d
23:01:37.650infoclickhouse7. DB::VolumeJBOD::reserve(unsigned long) @ 0x0000000010babf7c
23:01:37.650infoclickhouse6. DB::DiskLocal::reserve(unsigned long) @ 0x000000001093d64a
23:01:37.650infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:37.650infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:37.650infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:37.650infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:37.650infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:37.650infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:37.650infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:37.650infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:37.650infoclickhouseCannot add message to the log: Reserved 1.00 MiB on local disk `default`, having unreserved 23.68 GiB.
23:01:37.649infoclickhouse (version 24.8.14.39 (official build))
23:01:37.649infoclickhouse21. ? @ 0x000075475e85c353
23:01:37.649infoclickhouse20. ? @ 0x000075475e937609
23:01:37.649infoclickhouse19. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:37.649infoclickhouse18. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d2a2808
23:01:37.649infoclickhouse17. DB::SystemLog<DB::AsynchronousMetricLogElement>::savingThreadFunction() @ 0x00000000111a6cf3
23:01:37.649infoclickhouse16. DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x00000000125c8368
23:01:37.649infoclickhouse15. DB::PipelineExecutor::executeStepImpl(unsigned long, std::atomic<bool>*) @ 0x00000000125c8950
23:01:37.649infoclickhouse14. DB::ExecutionThreadContext::executeTask() @ 0x00000000125d4087
23:01:37.649infoclickhouse13. DB::ExceptionKeepingTransform::work() @ 0x0000000012838355
23:01:37.649infoclickhouse12. DB::runStep(std::function<void ()>, DB::ThreadStatus*, std::atomic<unsigned long>*) @ 0x0000000012838b2b
23:01:37.649infoclickhouse11. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>>(std::__function::__policy_storage const*) @ 0x0000000012838e18
23:01:37.649infoclickhouse10. DB::SinkToStorage::onConsume(DB::Chunk) @ 0x00000000129014b3
23:01:37.649infoclickhouse9. DB::MergeTreeSink::consume(DB::Chunk&) @ 0x00000000122e569c
23:01:37.649infoclickhouse8. DB::MergeTreeDataWriter::writeTempPartImpl(DB::BlockWithPartition&, std::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::shared_ptr<DB::Context const>, long, bool) @ 0x000000001217d863
23:01:37.649infoclickhouse7. DB::MergeTreeData::reserveSpacePreferringTTLRules(std::shared_ptr<DB::StorageInMemoryMetadata const> const&, unsigned long, DB::MergeTreeDataPartTTLInfos const&, long, unsigned long, bool, std::shared_ptr<DB::IDisk>) const @ 0x0000000011f9f859
23:01:37.649infoclickhouse6. DB::MergeTreeData::tryReserveSpacePreferringTTLRules(std::shared_ptr<DB::StorageInMemoryMetadata const> const&, unsigned long, DB::MergeTreeDataPartTTLInfos const&, long, unsigned long, bool, std::shared_ptr<DB::IDisk>) const @ 0x0000000011fa1f3e
23:01:37.649infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:37.649infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:37.649infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:37.649infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:37.649infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:37.648infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:37.648infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:37.648infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:37.648infoclickhouseCannot add message to the log: Trying to reserve 1.00 MiB using storage policy from min volume index 0
23:01:37.646infoclickhouse (version 24.8.14.39 (official build))
23:01:37.646infoclickhouse10. ? @ 0x000075475e85c353
23:01:37.646infoclickhouse9. ? @ 0x000075475e937609
23:01:37.646infoclickhouse8. void* std::__thread_proxy[abi:v15007]<std::tuple<std::unique_ptr<std::__thread_struct, std::default_delete<std::__thread_struct>>, void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda0'()>>(void*) @ 0x000000000d252d89
23:01:37.646infoclickhouse7. void std::__function::__policy_invoker<void ()>::__call_impl<std::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true, true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::'lambda'()&&)::'lambda'(), void ()>>(std::__function::__policy_storage const*) @ 0x000000000d2a2808
23:01:37.646infoclickhouse6. DB::SystemLog<DB::AsynchronousMetricLogElement>::savingThreadFunction() @ 0x00000000111a6431
23:01:37.646infoclickhouse5. DB::OwnSplitChannel::log(Poco::Message const&) @ 0x000000000d498070
23:01:37.646infoclickhouse4. DB::OwnSplitChannel::tryLogSplit(Poco::Message const&) @ 0x000000000d4981f9
23:01:37.646infoclickhouse3. DB::OwnFormattingChannel::logExtended(DB::ExtendedLogMessage const&) @ 0x000000000d4948ae
23:01:37.646infoclickhouse2. Poco::FileChannel::log(Poco::Message const&) @ 0x00000000152dad69
23:01:37.646infoclickhouse1. Poco::RotateBySizeStrategy::mustRotate(Poco::LogFile*) @ 0x00000000152ffce3
23:01:37.646infoclickhouse0. Poco::FileException::FileException(String const&, int) @ 0x00000000152d71e9
23:01:37.646infoclickhouse{"container_created_at":"2026-07-13T03:11:53.840600185Z","container_id":"af56201320c606f4c32ad827d92245c39330708d136407b4210b858237843392","container_name":"clickhouse","display":"","host":"f83ce4bf68a2","image":"clickhouse/clickhouse-server:24.8","label":{"com.clickhouse.build.version":"24.8.14.39","org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"20.04"},"level":"info","message":"","service":"clickhouse","source_type":"docker_logs","stream":"stderr"}
23:01:37.646infoclickhousePoco::Exception. Code: 1000, e.code() = 0, File access error: /var/log/clickhouse-server/clickhouse-server.log, Stack trace (when copying this message, always include the lines below):
23:01:37.646infoclickhouseCannot add message to the log: Flushing system log, 1743 entries to flush up to offset 598488906
23:01:37.626infoworkerGET / -> 200
23:01:37.597infopaymentsGET / -> 200
23:01:37.596infoworkerPOST /process -> 200