Index: /trunk/src/grep/Makefile.kmk
===================================================================
--- /trunk/src/grep/Makefile.kmk	(revision 3536)
+++ /trunk/src/grep/Makefile.kmk	(revision 3537)
@@ -56,14 +56,7 @@
 PROGRAMS += kmk_grep
 kmk_grep_TEMPLATE = BINC99
-kmk_grep_DEPS = \
-	$(kmk_grep_0_OUTDIR)/config.h \
-	$(kmk_grep_0_OUTDIR)/configmake.h
-kmk_grep_DEPS.win = \
-	$(kmk_grep_0_OUTDIR)/getopt-cdefs.h \
-	$(kmk_grep_0_OUTDIR)/getopt.h \
-	$(kmk_grep_0_OUTDIR)/unistd.h \
-       $(kmk_grep_0_OUTDIR)/sigsegv.h \
-       $(kmk_grep_0_OUTDIR)/fnmatch.h
-kmk_grep_CLEAN = $(kmk_grep_DEPS)
+kmk_grep_DEFS = \
+	KMK_GREP \
+	HAVE_CONFIG_H
 kmk_grep_CFLAGS.solaris = -std=gnu99
 kmk_grep_CFLAGS.win += -wd4018 # grep.c(797): warning C4018: '<': signed/unsigned mismatch [lots of size_t / ptrdiff_t mixups]
@@ -72,14 +65,11 @@
 kmk_grep_CFLAGS.win += -wd4146 # fnmatch_loop.c(1067): warning C4146: unary minus operator applied to unsigned type, result still unsigned
 kmk_grep_CFLAGS.win += -wd4308 # fnmatch_loop.c(1067): warning C4308: negative integral constant converted to unsigned type
+ifdef KBUILD_SOLARIS_10
+ kmk_grep_CFLAGS += -std=gnu99
+endif
 kmk_grep_INCS = \
 	$(kmk_grep_0_OUTDIR) \
 	. \
 	lib
-kmk_grep_DEFS = \
-	KMK_GREP \
-	HAVE_CONFIG_H
-ifdef KBUILD_SOLARIS_10
- kmk_grep_CFLAGS += -std=gnu99
-endif
 kmk_grep_SOURCES = \
 	src/grep.c \
@@ -153,55 +143,28 @@
 	../lib/nt/fts-nt.c
 
-#src/pcresearch.c
-
-#kmk_grep_SOURCES.darwin = \
-#	lib/strverscmp.c \
-#	lib/obstack.c \
-#	lib/getline.c
-#kmk_grep_SOURCES.dragonfly = \
-#	lib/strverscmp.c \
-#	lib/obstack.c \
-#	lib/getline.c
-#kmk_grep_SOURCES.freebsd = \
-#	lib/strverscmp.c \
-#	lib/obstack.c \
-#	lib/getline.c
-#kmk_grep_SOURCES.haiku = \
-#	lib/strverscmp.c \
-#	lib/obstack.c
-#kmk_grep_SOURCES.netbsd = \
-#	lib/strverscmp.c \
-#	lib/obstack.c \
-#	lib/getline.c
-#kmk_grep_SOURCES.openbsd = \
-#	lib/strverscmp.c \
-#	lib/obstack.c \
-#	lib/getline.c
-#kmk_grep_SOURCES.solaris = \
-#	lib/strverscmp.c \
-#	lib/obstack.c \
-#	lib/getline.c
-#kmk_grep_SOURCES.win = \
-#	lib/strverscmp.c \
-#	lib/obstack.c \
-#	lib/mkstemp.c \
-#	lib/getline.c \
-#	../lib/startuphacks-win.c
-
-#kmk_grep_LIBS.win = $(LIB_KUTIL) # for stdout optimizations.
-
-MAKEFILE_GREP := $(MAKEFILE_CURRENT)
-
-include $(FILE_KBUILD_SUB_FOOTER)
-
-#
+#TODO use:	../lib/startuphacks-win.c
+
+#
+# We generate a few files.
+#
+kmk_grep_DEPS = \
+	$(kmk_grep_0_OUTDIR)/config.h \
+	$(kmk_grep_0_OUTDIR)/configmake.h
+kmk_grep_DEPS.win = \
+	$(kmk_grep_0_OUTDIR)/getopt-cdefs.h \
+	$(kmk_grep_0_OUTDIR)/getopt.h \
+	$(kmk_grep_0_OUTDIR)/unistd.h \
+	$(kmk_grep_0_OUTDIR)/sigsegv.h \
+	$(kmk_grep_0_OUTDIR)/fnmatch.h
+
+kmk_grep_CLEAN = $(kmk_grep_DEPS)
+kmk_grep_CLEAN.win = $(kmk_grep_DEPS.win) $(kmk_grep_LNK_DEPS.win)
+
 # Use checked in config.h instead of running ./configure for it.
-#
-kmk_grep_config.h.$(KBUILD_TARGET) := $(kmk_grep_DEFPATH)/config.h.$(KBUILD_TARGET)
-
-$(kmk_grep_0_OUTDIR)/config.h: $(kmk_grep_DEFPATH)/config.$(KBUILD_TARGET).h | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+$$(kmk_grep_0_OUTDIR)/config.h: $(PATH_SUB_CURRENT)/config.$(KBUILD_TARGET).h | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(CP) -f -- "$^" "$@"
 
-$(kmk_grep_0_OUTDIR)/configmake.h: $(MAKEFILE_GREP) | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+# This shouldn't be used with NLS disabled, so the paths doesn't matter I hope.
+$$(kmk_grep_0_OUTDIR)/configmake.h: $(MAKEFILE_CURRENT) | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(APPEND) -tn "$@" \
 		'#define PREFIX         "kBuild/bin/$(KBUILD_TARGET).$(KBUILD_TARGET_ARCH)/"'; \
@@ -234,9 +197,7 @@
 		'#define PKGLIBEXECDIR  "kBuild/bin/$(KBUILD_TARGET).$(KBUILD_TARGET_ARCH)/"';
 
-#
-# Stuff Windows needs.
-#
-$(kmk_grep_0_OUTDIR)/getopt-cdefs.h: $(kmk_grep_DEFPATH)/lib/getopt-cdefs.in.h \
-		$(MAKEFILE_GREP) | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+# Windows needs a few more things:
+$$(kmk_grep_0_OUTDIR)/getopt-cdefs.h: $(PATH_SUB_CURRENT)/lib/getopt-cdefs.in.h \
+		$(MAKEFILE_CURRENT) | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(SED) \
        	-e 's,@HAVE_SYS_CDEFS_H@,0,' \
@@ -244,6 +205,6 @@
                $(qdeps sh,$@,1)
 
-$(kmk_grep_0_OUTDIR)/getopt.h: $(kmk_grep_DEFPATH)/lib/getopt.in.h \
-		$(MAKEFILE_GREP) | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+$$(kmk_grep_0_OUTDIR)/getopt.h: $(PATH_SUB_CURRENT)/lib/getopt.in.h \
+		$(MAKEFILE_CURRENT) | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(SED) \
        	-e 's,@GUARD_PREFIX@,GNULIB,' \
@@ -257,6 +218,6 @@
                $(qdeps sh,$@,1)
 
-$(kmk_grep_0_OUTDIR)/fnmatch.h: $(kmk_grep_DEFPATH)/lib/fnmatch.in.h \
-		$(MAKEFILE_GREP) | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+$$(kmk_grep_0_OUTDIR)/fnmatch.h: $(PATH_SUB_CURRENT)/lib/fnmatch.in.h \
+		$(MAKEFILE_CURRENT) | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(SED) \
        	-e 's,@GUARD_PREFIX@,GNULIB,' \
@@ -271,47 +232,37 @@
                $(qdeps sh,$@,1)
 
-$(kmk_grep_0_OUTDIR)/unistd.h: $(MAKEFILE_GREP) | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+$$(kmk_grep_0_OUTDIR)/unistd.h: $(MAKEFILE_CURRENT) | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(APPEND) -tn "$@" \
 		'extern int getpagesize(void);' \
 		'extern int fchdir(int);'
 
-$(kmk_grep_0_OUTDIR)/sigsegv.h: $(kmk_grep_DEFPATH)/lib/sigsegv.in.h | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+$$(kmk_grep_0_OUTDIR)/sigsegv.h: $(PATH_SUB_CURRENT)/lib/sigsegv.in.h | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(CP) -f -- "$^" "$@"
 
-# Not needed for VCC142:
-$(kmk_grep_0_OUTDIR)/inttypes.h: $(kmk_grep_DEFPATH)/lib/inttypes.in.h $(MAKEFILE_GREP) | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
-	$(SED) \
-       	-e '/@PRAGMA_SYSTEM_HEADER@/d' \
-       	-e '/@PRAGMA_COLUMNS@/d' \
-		-e 's,@HAVE_INTTYPES_H@,0,' \
-		-e '/@INCLUDE_NEXT@/d' \
-		-e 's,@APPLE_UNIVERSAL_BUILD@,0,' \
-		-e 's,@INT64_MAX_EQ_LONG_MAX@,0/*not-on-win*/,' \
-		-e 's,@INT64_MAX_EQ_ULONG_MAX@,0/*not-on-win*/,' \
-		-e 's,@INT32_MAX_LT_INTMAX_MAX@,0,' \
-		-e 's,@UINT32_MAX_LT_UINTMAX_MAX@,0,' \
-		-e 's,@UINT64_MAX_EQ_ULONG_MAX@,1,' \
-		-e 's,@PRIPTR_PREFIX@,"ll" /*fixme*/,' \
-		-e 's,@GNULIB_IMAXABS@,0,' \
-		-e 's,@HAVE_DECL_IMAXABS@,0,' \
-		-e 's,@GNULIB_IMAXDIV@,0,' \
-		-e 's,@HAVE_IMAXDIV_T@,0,' \
-		-e 's,@HAVE_DECL_IMAXDIV@,0,' \
-		-e 's,@GNULIB_STRTOIMAX@,0,' \
-		-e 's,@REPLACE_STRTOIMAX@,0,' \
-		-e 's,@HAVE_DECL_STRTOIMAX@,0,' \
-		-e 's,@GNULIB_STRTOUMAX@,0,' \
-		-e 's,@REPLACE_STRTOUMAX@,0,' \
-		-e 's,@HAVE_DECL_STRTOUMAX@,0,' \
-		--output "$@" \
-               $(qdeps sh,$@,1)
-
-$(kmk_grep_0_OUTDIR)/stdbool.h: $(MAKEFILE_GREP) | $(call DIRDEP,$(kmk_grep_0_OUTDIR))
+if 1 # Add a manifest making UTF-8 as the active code page.
+kmk_grep_LNK_DEPS.win = \
+	$(kmk_grep_0_OUTDIR)/kmk_grep.manifest
+
+$$(kmk_grep_0_OUTDIR)/kmk_grep.manifest: $(MAKEFILE_CURRENT) | $$(call DIRDEP,$$(kmk_grep_0_OUTDIR))
 	$(APPEND) -tn "$@" \
-		'#ifndef ___STDBOOL_H___' \
-		'#define ___STDBOOL_H___' \
-		'typedef unsigned char bool;' \
-		'#define true  (1)' \
-		'#define false (0)' \
-		'#endif'
-
+		'<?xml version="1.0" encoding="UTF-8" standalone="yes"?>' \
+		'<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">' \
+		'  <assemblyIdentity type="win32" name="kmk_grep.exe" version="3.7.0.0"/>' \
+		'  <application>' \
+		'    <windowsSettings>' \
+		'      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>' \
+		'    </windowsSettings>' \
+		'  </application>' \
+		'</assembly>'
+
+# HACK ALERT! Using kmk_builting_redirect here to bypass some shell quoting
+#             issue.  Also, no idea why we need to escape the hash (\#1).
+kmk_grep_POST_CMDS.win = \
+	$(REDIRECT) -- $(PATH_SDK_WINSDK10-UM_BIN)/mt.exe \
+		-manifest "$(subst /,\\,$(kmk_grep_0_OUTDIR)/kmk_grep.manifest)" \
+		'-outputresource:$(subst /,\\,$(out));\#1'
+endif
+
+
+include $(FILE_KBUILD_SUB_FOOTER)
+
Index: /trunk/src/grep/src/grep.c
===================================================================
--- /trunk/src/grep/src/grep.c	(revision 3536)
+++ /trunk/src/grep/src/grep.c	(revision 3537)
@@ -443,8 +443,31 @@
 static int stdout_errno;
 
-#if defined(KMK_GREP) && defined(KBUILD_OS_WINDOWS)
-# include <assert.h>
+#ifdef KMK_GREP
+# ifdef KBUILD_OS_WINDOWS
+#  include <assert.h>
 static void fwrite_errno (void const *, size_t, size_t);
+static int g_fStdOutIsConsole = -1; /* TRUE or FALSE; -1 if not initialize. */
 #endif
+
+/* Attempts to set the code page, leave the rest of the locale as default. */
+static void kmk_grep_set_codepage (const char *pszCodepage)
+{
+# ifdef KBUILD_OS_WINDOWS
+  /* Make sure it starts with a dot: */
+  char szDot[256];
+  if (pszCodepage[0] != '.')
+    {
+      snprintf (szDot, sizeof(szDot), ".%s", pszCodepage);
+      pszCodepage = szDot;
+    }
+
+  if (setlocale (LC_ALL, pszCodepage) == NULL)
+    error (0, errno, _("warning: setlocale (LC_ALL, \"%s\") failed"),
+           pszCodepage);
+
+  g_fStdOutIsConsole = -1; /* ensure this is reinitialized. */
+# endif
+}
+#endif /* KMK_GREP */
 
 static void
@@ -494,18 +517,71 @@
   /*
    * This trick reduces the runtime of 'grep -r GNU .' in the grep source dir
-   * from just above 11 seconds to just below 1.2 seconds.
-   * Note! s_is_console is for keeping output to file speedy.
+   * from just above 11 seconds to around 0.8 seconds.
+   *
+   * The trouble with the microsoft CRTs (both the old and the new UCRT), is
+   * that we end up writing one char at the time when writing to the console,
+   * which is a total performance killer. write_double_translated_ansi_nolock()
+   * and write_requires_double_translation_nolock() in lowio/write.cpp in the
+   * UCRT sources have further details.
    */
-  static int s_is_console = -1;
-  if (s_is_console != -1)
+  static HANDLE s_hStdOut = INVALID_HANDLE_VALUE;
+  if (g_fStdOutIsConsole != -1)
     { /* likely*/ }
   else
-    s_is_console = is_console (fileno (stdout));
-  if (  s_is_console
-      ? maybe_con_fwrite (ptr, size, nmemb, stdout) != nmemb
-      : fwrite (ptr, size, nmemb, stdout) != nmemb)
-#else
+    {
+      DWORD fModeIgnored;
+      s_hStdOut          = (HANDLE)_get_osfhandle (fileno (stdout));
+      g_fStdOutIsConsole = GetConsoleMode (s_hStdOut, &fModeIgnored)
+                         ? TRUE : FALSE;
+      if (getenv("KMK_GREP_CONSOLE_DEBUG"))
+        fprintf(stderr, "kmk_grep: g_fStdOutIsConsole=%d s_hStdOut=%p codepage=%u\n",
+                g_fStdOutIsConsole, s_hStdOut, ___lc_codepage_func());
+    }
+  if (g_fStdOutIsConsole == TRUE && size && nmemb)
+    {
+      size_t const cbToWrite = size * nmemb;
+      if (   cbToWrite < (size_t)INT_MAX / 4
+          && cbToWrite >= size
+          && cbToWrite >= nmemb)
+        {
+          /* ASSUME that one input byte won't be translated to more than one
+             surrogate pair, or two compound UTF-16 codepoints. */
+          wchar_t        awcBuf[1024];
+          wchar_t       *pawcFree = NULL;
+          wchar_t       *pawcBuf;
+          size_t         cwcBuf = cbToWrite * 2 + 16;
+          if (cwcBuf < sizeof(awcBuf) / sizeof(wchar_t))
+            {
+              cwcBuf = sizeof(awcBuf) / sizeof(wchar_t);
+              pawcBuf = awcBuf;
+            }
+          else
+              pawcFree = pawcBuf = (wchar_t *)malloc(cwcBuf * sizeof(wchar_t));
+          if (pawcBuf)
+            {
+              int cwcToWrite = MultiByteToWideChar(___lc_codepage_func(),
+                                                   0 /*dwFlags*/,
+                                                   ptr, (int)cbToWrite,
+                                                   pawcBuf, (int)(cwcBuf - 1));
+              if (cwcToWrite > 0)
+                {
+                  pawcBuf[cwcToWrite] = '\0';
+
+                  /* Let the CRT do the rest.  At least the Visual C++ 2010 CRT
+                     sources indicates _cputws will do the right thing.  */
+                  fflush(stdout);
+                  int rc = _cputws(pawcBuf);
+                  if (pawcFree)
+                    free(pawcFree);
+                  if (rc != 0)
+                    stdout_errno = errno;
+                  return;
+                }
+              free(pawcFree);
+            }
+        }
+    }
+#endif
   if (fwrite (ptr, size, nmemb, stdout) != nmemb)
-#endif
     stdout_errno = errno;
 }
@@ -536,4 +612,8 @@
   LINE_BUFFERED_OPTION,
   LABEL_OPTION,
+#ifdef KMK_GREP
+  UTF8_OPTION,
+  CODEPAGE_OPTION,
+#endif
   NO_IGNORE_CASE_OPTION
 };
@@ -593,4 +673,9 @@
   {"with-filename", no_argument, NULL, 'H'},
   {"word-regexp", no_argument, NULL, 'w'},
+#ifdef KMK_GREP
+  {"utf8", no_argument, NULL, UTF8_OPTION},
+  {"cp", required_argument, NULL, CODEPAGE_OPTION},
+  {"codepage", required_argument, NULL, CODEPAGE_OPTION},
+#endif
   {0, 0, 0, 0}
 };
@@ -2097,4 +2182,13 @@
   -U, --binary              do not strip CR characters at EOL (MSDOS/Windows)\n\
 \n"));
+#ifdef KMK_GREP
+      printf (_("\
+kmk_grep extensions:\n\
+  --codepage=NUM            switches the locale to the given codepage, \n\
+                            affecting how input files are treated and outputted\n\
+                            windows only, ignored elsewhere\n\
+  --utf8                    shorthand for --codepage=UTF8\n\
+\n"));
+#endif
       printf (_("\
 When FILE is '-', read standard input.  With no FILE, read '.' if\n\
@@ -2541,5 +2635,10 @@
   /* Internationalization. */
 #if defined HAVE_SETLOCALE
-  setlocale (LC_ALL, "");
+# if defined(KMK_GREP) && defined(KBUILD_OS_WINDOWS)
+  if (getenv ("KMK_GREP_CODEPAGE"))
+    kmk_grep_set_codepage (getenv ("KMK_GREP_CODEPAGE"));
+  else
+# endif
+    setlocale (LC_ALL, "");
 #endif
 #if defined ENABLE_NLS
@@ -2860,4 +2959,20 @@
         break;
 
+#ifdef KMK_GREP
+      /* The --utf8 and --codepage <cp> options are mainly for windows where
+         UCRT doesn't check any of the standard locale selecting environment
+         variables and we have to give it directly to setlocale if we want
+         any control beyond the Windows defaults.
+
+         The UCRT setlocale has a nice feature of allowing us to set just
+         the codepage, omitting the rest of the locale spec. */
+      case UTF8_OPTION:
+        kmk_grep_set_codepage (".UTF-8");
+        break;
+      case CODEPAGE_OPTION:
+        kmk_grep_set_codepage (optarg);
+        break;
+#endif
+
       case 0:
         /* long options */
