stuff

tools/server/server-context.cpp

@@ -38,6 +38,7 @@
 using json = nlohmann::ordered_json;
 
 constexpr int HTTP_POLLING_SECONDS = 1;
+constexpr int64_t PROMPT_TIMING_LOG_INTERVAL_US = 5*1000*1000;
 
 static uint32_t server_n_outputs_max(const common_params & params) {
     const uint32_t n_batch  = params.n_batch;
@@ -276,6 +277,7 @@ struct server_slot {
     // TODO @ngxson : move all metrics to a sub-struct for clarity
     int64_t t_start_process_prompt;
     int64_t t_start_generation;
+    int64_t t_prompt_print_last = 0;
     int64_t t_print_last = 0;
     int32_t n_decoded_last = 0;
 
@@ -569,13 +571,17 @@ struct server_slot {
         SLT_INF(*this, "n_decoded = %6d, tg = %6.2f t/s, tg_3s = %6.2f t/s\n", n_decoded, n_gen_second, n_gen_second_win);
     }
 
-    void print_timings_pp() const {
+    void print_timings_pp() {
+        const int64_t t_now = ggml_time_us();
+
+        if (t_now - t_prompt_print_last < PROMPT_TIMING_LOG_INTERVAL_US) {
+            return;
+        }
+
         const double n_prompt_second = 1e3 / t_prompt_processing * n_prompt_tokens_processed;
         const double f_progress = (float) prompt.n_tokens() / task->n_tokens();
 
-        if (t_prompt_processing < 3000.0) {
-            return;
-        }
+        t_prompt_print_last = t_now;
 
         SLT_INF(*this, "prompt processing, n_tokens = %6d, progress = %.2f, t = %6.2f s / %.2f tokens per second\n",
                 n_prompt_tokens_processed, f_progress, t_prompt_processing / 1e3, n_prompt_second);
@@ -3072,6 +3078,7 @@ private:
                     // TODO: maybe move branch to outside of this loop in the future
                     if (slot.state == SLOT_STATE_STARTED) {
                         slot.t_start_process_prompt = ggml_time_us();
+                        slot.t_prompt_print_last = slot.t_start_process_prompt;
                         slot.t_start_generation = 0;
 
                         slot.state = SLOT_STATE_PROCESSING_PROMPT;