Changeset 3191 for trunk/src

Timestamp:

Mar 26, 2018, 11:16:06 AM (7 years ago)

Author:

bird

Message:

kmk/output: Memory buffering fixes.

Location:

trunk/src/kmk

Files:

• output.c (modified) (6 diffs)
• w32/winchildren.c (modified) (1 diff)

trunk/src/kmk/output.c

@@ -67 +67 @@
                                - offsetof (struct output_segment, runs) \
                                - sizeof (struct output_run))
+# define MEMBUF_MAX_TOTAL     (  sizeof (void *) <= 4 \
+                               ? (size_t)512*1024 : (size_t)16*1024*1024 )
 
 static void *acquire_semaphore (void);
@@ -220 +222 @@
 }
 
-/* Helper for membuf_new_segment_write that figures out now much data needs to
-   be moved from the previous run in order to make it end with a newline.  */
+/* Helper for membuf_write_new_segment and membuf_dump_most that figures out
+   now much data needs to be moved from the previous run in order to make it
+   end with a newline.  */
 static size_t membuf_calc_move_len (struct output_run *tail_run)
 {
@@ -242 +245 @@
    a newline so that we pass whole lines to fwrite when dumping. */
 static size_t
-membuf_new_segment_write (struct output_membuf *membuf, const char *src,
+membuf_write_new_segment (struct output_membuf *membuf, const char *src,
                           size_t len, unsigned int *pseqno)
 {
@@ -280 +283 @@
   membuf->tail_run = &new_seg->runs[0];
   membuf->total += segsize;
-  membuf->left = segsize - sizeof (struct output_run);
+  membuf->left = segsize - sizeof (struct output_run) - offset_runs;
 
   /* Initialize and write data to the first run. */
@@ -314 +317 @@
 }
 
-/* Worker for output_write that will try dump as much as possible of the
-   output, but making sure to try leave unfinished lines. */
+/* Worker for output_write that will dump most of the output when we hit
+   MEMBUF_MAX_TOTAL on either of the two membuf structures, then free all the
+   output segments.  Incomplete lines will be held over to the next buffers
+   and copied into new segments. */
 static void
 membuf_dump_most (struct output *out)
 {
-  /** @todo check last segment and make local copies.   */
-  membuf_dump (out);
+  size_t out_to_move = membuf_calc_move_len (out->out.tail_run);
+  size_t err_to_move = membuf_calc_move_len (out->err.tail_run);
+  if (!out_to_move && !err_to_move)
+    membuf_dump (out);
+  else
+    {
+      /* Allocate a stack buffer for holding incomplete lines.  This should be
+         fine since we're only talking about max 2 * MEMBUF_MAX_MOVE_LEN.
+         The -1 on the sequence numbers, ise because membuf_write_new_segment
+         will increment them before use. */
+      unsigned int out_seqno = out_to_move ? out->out.tail_run->seqno - 1 : 0;
+      unsigned int err_seqno = err_to_move ? out->err.tail_run->seqno - 1 : 0;
+      char *tmp = alloca (out_to_move + err_to_move);
+      if (out_to_move)
+        {
+          out->out.tail_run->len -= out_to_move;
+          memcpy (tmp,
+                  (char *)(out->out.tail_run + 1) + out->out.tail_run->len,
+                  out_to_move);
+        }
+      if (err_to_move)
+        {
+          out->err.tail_run->len -= err_to_move;
+          memcpy (tmp + out_to_move,
+                  (char *)(out->err.tail_run + 1) + out->err.tail_run->len,
+                  err_to_move);
+        }
+
+      membuf_dump (out);
+
+      if (out_to_move)
+        {
+          size_t written = membuf_write_new_segment (&out->out, tmp,
+                                                     out_to_move, &out_seqno);
+          assert (written == out_to_move); (void)written;
+        }
+      if (err_to_move)
+        {
+          size_t written = membuf_write_new_segment (&out->err,
+                                                     tmp + out_to_move,
+                                                     err_to_move, &err_seqno);
+          assert (written == err_to_move); (void)written;
+        }
+    }
 }
 
@@ -353 +400 @@
           if (!runlen)
             {
-              size_t max_total = sizeof (membuf) <= 4 ? 512*1024 : 16*1024*1024;
-              if (membuf->total < max_total)
-                runlen = membuf_new_segment_write (membuf, src, len, &out->seqno);
+              if (membuf->total < MEMBUF_MAX_TOTAL)
+                runlen = membuf_write_new_segment (membuf, src, len, &out->seqno);
               else
                 membuf_dump_most (out);

trunk/src/kmk/w32/winchildren.c

@@ -677 +677 @@
         /* Move offPendingRead ahead by cbRead. */
         pPipe->offPendingRead += cbNewData;
-        assert(pPipe->offPendingRead < pPipe->cbBuffer);
+        assert(pPipe->offPendingRead <= pPipe->cbBuffer);
         if (pPipe->offPendingRead > pPipe->cbBuffer)
             pPipe->offPendingRead = pPipe->cbBuffer;