Cygwin: pty: add pseudo console support.

- Support pseudo console in PTY. Pseudo console is a new feature
  in Windows 10 1809, which provides console APIs on virtual
  terminal. With this patch, native console applications can work
  in PTYs such as mintty, ssh, gnu screen or tmux.

winsup/cygwin/dtable.cc

@@ -147,6 +147,36 @@ dtable::get_debugger_info ()
 void
 dtable::stdio_init ()
 {
+  bool need_fixup_handle = false;
+  fhandler_pty_slave *ptys = NULL;
+  bool is_pty[3] = {false, false, false};
+  for (int fd = 0; fd < 3; fd ++)
+    {
+      fhandler_base *fh = cygheap->fdtab[fd];
+      if (fh && fh->get_major () == DEV_PTYS_MAJOR)
+	{
+	  ptys = (fhandler_pty_slave *) fh;
+	  if (ptys->getPseudoConsole ())
+	    {
+	      is_pty[fd] = true;
+	      bool attached = !!fhandler_console::get_console_process_id
+		(ptys->getHelperProcessId (), true);
+	      if (!attached)
+		{
+		  /* Not attached to pseudo console in fork() or spawn()
+		     by some reason. This happens if the executable is
+		     a windows GUI binary, such as mintty. */
+		  FreeConsole ();
+		  AttachConsole (ptys->getHelperProcessId ());
+		  need_fixup_handle = true;
+		}
+	      ptys->reset_switch_to_pcon ();
+	    }
+	}
+    }
+  if (need_fixup_handle)
+    goto fixup_handle;
+
   if (myself->cygstarted || ISSTATE (myself, PID_CYGPARENT))
     {
       tty_min *t = cygwin_shared->tty.get_cttyp ();
@@ -155,6 +185,27 @@ dtable::stdio_init ()
       return;
     }
 
+fixup_handle:
+  if (need_fixup_handle)
+    {
+      HANDLE h;
+      h = CreateFile ("CONIN$", GENERIC_READ, FILE_SHARE_READ,
+		       NULL, OPEN_EXISTING, 0, 0);
+      if (is_pty[0])
+	{
+	  SetStdHandle (STD_INPUT_HANDLE, h);
+	  ptys->set_handle (h);
+	}
+      h = CreateFile ("CONOUT$", GENERIC_WRITE, FILE_SHARE_WRITE,
+		       NULL, OPEN_EXISTING, 0, 0);
+      if (is_pty[1])
+	SetStdHandle (STD_OUTPUT_HANDLE, h);
+      if (is_pty[2])
+	SetStdHandle (STD_ERROR_HANDLE, h);
+      if (is_pty[1] || is_pty[2])
+	ptys->set_output_handle (h);
+    }
+
   HANDLE in = GetStdHandle (STD_INPUT_HANDLE);
   HANDLE out = GetStdHandle (STD_OUTPUT_HANDLE);
   HANDLE err = GetStdHandle (STD_ERROR_HANDLE);

winsup/cygwin/fhandler.h

@@ -2019,6 +2019,7 @@ private:
   static bool need_invisible ();
   static void free_console ();
   static const char *get_nonascii_key (INPUT_RECORD& input_rec, char *);
+  static DWORD get_console_process_id (DWORD pid, bool match);
 
   fhandler_console (void *) {}
 
@@ -2051,8 +2052,8 @@ class fhandler_pty_common: public fhandler_termios
  public:
   fhandler_pty_common ()
     : fhandler_termios (),
-      output_mutex (NULL),
-    input_mutex (NULL), input_available_event (NULL)
+    output_mutex (NULL), input_mutex (NULL),
+    input_available_event (NULL)
   {
     pc.file_attributes (FILE_ATTRIBUTE_NORMAL);
   }
@@ -2089,14 +2090,29 @@ class fhandler_pty_common: public fhandler_termios
     return fh;
   }
 
+  bool attach_pcon_in_fork (void)
+  {
+    return get_ttyp ()->attach_pcon_in_fork;
+  }
+  DWORD getHelperProcessId (void)
+  {
+    return get_ttyp ()->HelperProcessId;
+  }
+  HPCON getPseudoConsole (void)
+  {
+    return get_ttyp ()->hPseudoConsole;
+  }
+
  protected:
-  BOOL process_opost_output (HANDLE h, const void *ptr, ssize_t& len, bool is_echo);
+  BOOL process_opost_output (HANDLE h,
+			     const void *ptr, ssize_t& len, bool is_echo);
+  bool check_switch_to_pcon (void);
 };
 
 class fhandler_pty_slave: public fhandler_pty_common
 {
   HANDLE inuse;			// used to indicate that a tty is in use
-  HANDLE output_handle_cyg;
+  HANDLE output_handle_cyg, io_handle_cyg;
 
   /* Helper functions for fchmod and fchown. */
   bool fch_open_handles (bool chown);
@@ -2106,9 +2122,13 @@ class fhandler_pty_slave: public fhandler_pty_common
  public:
   /* Constructor */
   fhandler_pty_slave (int);
+  /* Destructor */
+  ~fhandler_pty_slave ();
 
   void set_output_handle_cyg (HANDLE h) { output_handle_cyg = h; }
   HANDLE& get_output_handle_cyg () { return output_handle_cyg; }
+  void set_handle_cyg (HANDLE h) { io_handle_cyg = h; }
+  HANDLE& get_handle_cyg () { return io_handle_cyg; }
 
   int open (int flags, mode_t mode = 0);
   void open_setup (int flags);
@@ -2149,6 +2169,15 @@ class fhandler_pty_slave: public fhandler_pty_common
     copyto (fh);
     return fh;
   }
+  void set_switch_to_pcon (void);
+  void reset_switch_to_pcon (void);
+  void push_to_pcon_screenbuffer (const char *ptr, size_t len);
+  bool has_master_opened (void);
+  void mask_switch_to_pcon (bool mask)
+  {
+    get_ttyp ()->mask_switch_to_pcon = mask;
+  }
+  void fixup_after_attach (bool native_maybe);
 };
 
 #define __ptsname(buf, unit) __small_sprintf ((buf), "/dev/pty%d", (unit))
@@ -2157,17 +2186,17 @@ class fhandler_pty_master: public fhandler_pty_common
   int pktmode;			// non-zero if pty in a packet mode.
   HANDLE master_ctl;		// Control socket for handle duplication
   cygthread *master_thread;	// Master control thread
-  HANDLE from_master, to_master;
+  HANDLE from_master, to_master, from_slave, to_slave;
   HANDLE echo_r, echo_w;
   DWORD dwProcessId;		// Owner of master handles
-  HANDLE io_handle_cyg, to_master_cyg;
+  HANDLE to_master_cyg, from_master_cyg;
   cygthread *master_fwd_thread;	// Master forwarding thread
 
 public:
   HANDLE get_echo_handle () const { return echo_r; }
-  HANDLE& get_handle_cyg () { return io_handle_cyg; }
   /* Constructor */
   fhandler_pty_master (int);
+  ~fhandler_pty_master ();
 
   DWORD pty_master_thread ();
   DWORD pty_master_fwd_thread ();
@@ -2212,6 +2241,8 @@ public:
     copyto (fh);
     return fh;
   }
+
+  bool setup_pseudoconsole (void);
 };
 
 class fhandler_dev_null: public fhandler_base

winsup/cygwin/fhandler_console.cc

@@ -1056,6 +1056,19 @@ fhandler_console::close ()
 
   CloseHandle (get_handle ());
   CloseHandle (get_output_handle ());
+
+  /* If already attached to pseudo console, don't call free_console () */
+  cygheap_fdenum cfd (false);
+  while (cfd.next () >= 0)
+    if (cfd->get_major () == DEV_PTYM_MAJOR ||
+	cfd->get_major () == DEV_PTYS_MAJOR)
+      {
+	fhandler_pty_common *t =
+	  (fhandler_pty_common *) (fhandler_base *) cfd;
+	if (get_console_process_id (t->getHelperProcessId (), true))
+	  return 0;
+      }
+
   if (!have_execed)
     free_console ();
   return 0;
@@ -3119,6 +3132,25 @@ fhandler_console::need_invisible ()
   return b;
 }
 
+DWORD
+fhandler_console::get_console_process_id (DWORD pid, bool match)
+{
+  DWORD tmp;
+  int num = GetConsoleProcessList (&tmp, 1);
+  DWORD *list = (DWORD *)
+	      HeapAlloc (GetProcessHeap (), 0, num * sizeof (DWORD));
+  num = GetConsoleProcessList (list, num);
+  tmp = 0;
+  for (int i=0; i<num; i++)
+    if ((match && list[i] == pid) || (!match && list[i] != pid))
+      {
+	tmp = list[i];
+	//break;
+      }
+  HeapFree (GetProcessHeap (), 0, list);
+  return tmp;
+}
+
 DWORD
 fhandler_console::__acquire_input_mutex (const char *fn, int ln, DWORD ms)
 {

winsup/cygwin/fork.cc

@@ -134,6 +134,30 @@ child_info::prefork (bool detached)
 int __stdcall
 frok::child (volatile char * volatile here)
 {
+  cygheap_fdenum cfd (false);
+  while (cfd.next () >= 0)
+    if (cfd->get_major () == DEV_PTYM_MAJOR)
+      {
+	fhandler_base *fh = cfd;
+	fhandler_pty_master *ptym = (fhandler_pty_master *) fh;
+	if (ptym->getPseudoConsole () &&
+	    !fhandler_console::get_console_process_id (
+				ptym->getHelperProcessId (), true))
+
+	  {
+	    debug_printf ("found a PTY master %d: helper_PID=%d",
+			  ptym->get_minor (), ptym->getHelperProcessId ());
+	    if (ptym->attach_pcon_in_fork ())
+	      {
+		FreeConsole ();
+		if (!AttachConsole (ptym->getHelperProcessId ()))
+		  /* Error */;
+		else
+		  break;
+	      }
+	  }
+      }
+
   HANDLE& hParent = ch.parent;
 
   sync_with_parent ("after longjmp", true);

winsup/cygwin/init.cc

@@ -100,6 +100,7 @@ dll_entry (HANDLE h, DWORD reason, void *static_load)
 	 will always fall back to __global_locale, rather then crash due to
 	 _REENT->_locale having an arbitrary value. */
       alloca_dummy = alloca (CYGTLS_PADSIZE);
+      ZeroMemory (alloca_dummy, CYGTLS_PADSIZE);
       memcpy (_REENT, _GLOBAL_REENT, sizeof (struct _reent));
 
       dll_crt0_0 ();

winsup/cygwin/select.cc

@@ -667,6 +667,9 @@ peek_pipe (select_record *s, bool from_select)
 	    fhm->flush_to_slave ();
 	  }
 	  break;
+	case DEV_PTYS_MAJOR:
+	  ((fhandler_pty_slave *) fh)->reset_switch_to_pcon ();
+	  break;
 	default:
 	  if (fh->get_readahead_valid ())
 	    {
@@ -1178,17 +1181,32 @@ verify_tty_slave (select_record *me, fd_set *readfds, fd_set *writefds,
   return set_bits (me, readfds, writefds, exceptfds);
 }
 
+static int
+pty_slave_startup (select_record *s, select_stuff *)
+{
+  fhandler_base *fh = (fhandler_base *) s->fh;
+  ((fhandler_pty_slave *) fh)->mask_switch_to_pcon (true);
+  return 1;
+}
+
+static void
+pty_slave_cleanup (select_record *s, select_stuff *)
+{
+  fhandler_base *fh = (fhandler_base *) s->fh;
+  ((fhandler_pty_slave *) fh)->mask_switch_to_pcon (false);
+}
+
 select_record *
 fhandler_pty_slave::select_read (select_stuff *ss)
 {
   select_record *s = ss->start.next;
   s->h = input_available_event;
-  s->startup = no_startup;
+  s->startup = pty_slave_startup;
   s->peek = peek_pipe;
   s->verify = verify_tty_slave;
   s->read_selected = true;
   s->read_ready = false;
-  s->cleanup = NULL;
+  s->cleanup = pty_slave_cleanup;
   return s;
 }
 

winsup/cygwin/spawn.cc

@@ -259,6 +259,8 @@ child_info_spawn::worker (const char *prog_arg, const char *const *argv,
 {
   bool rc;
   int res = -1;
+  DWORD pidRestore = 0;
+  bool attach_to_pcon = false;
 
   /* Check if we have been called from exec{lv}p or spawn{lv}p and mask
      mode to keep only the spawn mode. */
@@ -572,6 +574,58 @@ child_info_spawn::worker (const char *prog_arg, const char *const *argv,
 			 PROCESS_QUERY_LIMITED_INFORMATION))
 	sa = &sec_none_nih;
 
+      /* Attach to pseudo console if pty salve is used */
+      pidRestore = fhandler_console::get_console_process_id
+	(GetCurrentProcessId (), false);
+      fhandler_pty_slave *ptys = NULL;
+      for (int fd = 0; fd < 3; fd ++)
+	{
+	  fhandler_base *fh = ::cygheap->fdtab[fd];
+	  if (fh && fh->get_major () == DEV_PTYS_MAJOR)
+	    {
+	      ptys = (fhandler_pty_slave *) fh;
+	      if (ptys->getPseudoConsole () &&
+		  !fhandler_console::get_console_process_id (
+				     ptys->getHelperProcessId (), true))
+		{
+		  DWORD dwHelperProcessId = ptys->getHelperProcessId ();
+		  debug_printf ("found a PTY slave %d: helper_PID=%d",
+				fh->get_minor (), dwHelperProcessId);
+		  FreeConsole ();
+		  if (!AttachConsole (dwHelperProcessId))
+		    {
+		      /* Fallback */
+		      DWORD target[3] = {
+			STD_INPUT_HANDLE,
+			STD_OUTPUT_HANDLE,
+			STD_ERROR_HANDLE
+		      };
+		      if (fd == 0)
+			{
+			  ptys->set_handle (ptys->get_handle_cyg ());
+			  SetStdHandle (target[fd],
+					ptys->get_handle ());
+			}
+		      else
+			{
+			  ptys->set_output_handle (
+				       ptys->get_output_handle_cyg ());
+			  SetStdHandle (target[fd],
+					ptys->get_output_handle ());
+			}
+		    }
+		  else
+		    {
+		      init_console_handler (true);
+		      attach_to_pcon = true;
+		      break;
+		    }
+		}
+	    }
+	}
+      if (ptys)
+	ptys->fixup_after_attach (true);
+
     loop:
       /* When ruid != euid we create the new process under the current original
 	 account and impersonate in child, this way maintaining the different
@@ -869,6 +923,13 @@ child_info_spawn::worker (const char *prog_arg, const char *const *argv,
   this->cleanup ();
   if (envblock)
     free (envblock);
+
+  if (attach_to_pcon && pidRestore)
+    {
+      FreeConsole ();
+      AttachConsole (pidRestore);
+    }
+
   return (int) res;
 }
 

winsup/cygwin/strace.cc

@@ -279,6 +279,30 @@ strace::vprntf (unsigned category, const char *func, const char *fmt, va_list ap
 	      CloseHandle (h);
 	    }
 	}
+#if 1 /* Experimental code */
+      /* PTY with pseudo console cannot display data written to
+	 STD_ERROR_HANDLE (output_handle) if the process is cygwin
+	 process. output_handle works only in native console apps.
+	 Therefore the data should be written to output_handle_cyg
+	 as well. */
+      fhandler_base *fh = ::cygheap->fdtab[2];
+      if (fh && fh->get_major () == DEV_PTYS_MAJOR)
+	{
+	  fhandler_pty_slave *ptys = (fhandler_pty_slave *) fh;
+	  if (ptys->getPseudoConsole ())
+	    {
+	      HANDLE h_cyg = ptys->get_output_handle_cyg ();
+	      if (buf[len-1] == '\n' && len < NT_MAX_PATH - 1)
+		{
+		  buf[len-1] = '\r';
+		  buf[len] = '\n';
+		  len ++;
+		}
+	      WriteFile (h_cyg, buf, len, &done, 0);
+	      FlushFileBuffers (h_cyg);
+	    }
+	}
+#endif
     }
 
 #ifndef NOSTRACE

winsup/cygwin/tty.cc

@@ -234,7 +234,15 @@ tty::init ()
   was_opened = false;
   master_pid = 0;
   is_console = false;
+  attach_pcon_in_fork = false;
+  hPseudoConsole = NULL;
   column = 0;
+  switch_to_pcon = false;
+  screen_alternated = false;
+  mask_switch_to_pcon = false;
+  pcon_pid = 0;
+  num_pcon_attached_slaves = 0;
+  TermCodePage = 20127; /* ASCII */
 }
 
 HANDLE

winsup/cygwin/tty.h

@@ -28,6 +28,8 @@ details. */
 #define MIN_CTRL_C_SLOP 50
 #endif
 
+typedef void *HPCON;
+
 #include <devices.h>
 class tty_min
 {
@@ -88,14 +90,28 @@ public:
 
 private:
   HANDLE _from_master;
+  HANDLE _from_master_cyg;
   HANDLE _to_master;
   HANDLE _to_master_cyg;
+  HPCON hPseudoConsole;
+  HANDLE hHelperProcess;
+  DWORD HelperProcessId;
+  HANDLE hHelperGoodbye;
+  bool attach_pcon_in_fork;
+  bool switch_to_pcon;
+  bool screen_alternated;
+  bool mask_switch_to_pcon;
+  pid_t pcon_pid;
+  int num_pcon_attached_slaves;
+  UINT TermCodePage;
 
 public:
-  HANDLE from_master() const { return _from_master; }
-  HANDLE to_master() const { return _to_master; }
-  HANDLE to_master_cyg() const { return _to_master_cyg; }
+  HANDLE from_master () const { return _from_master; }
+  HANDLE from_master_cyg () const { return _from_master_cyg; }
+  HANDLE to_master () const { return _to_master; }
+  HANDLE to_master_cyg () const { return _to_master_cyg; }
   void set_from_master (HANDLE h) { _from_master = h; }
+  void set_from_master_cyg (HANDLE h) { _from_master_cyg = h; }
   void set_to_master (HANDLE h) { _to_master = h; }
   void set_to_master_cyg (HANDLE h) { _to_master_cyg = h; }
 
@@ -117,7 +133,9 @@ public:
   void set_master_ctl_closed () {master_pid = -1;}
   static void __stdcall create_master (int);
   static void __stdcall init_session ();
+  friend class fhandler_pty_common;
   friend class fhandler_pty_master;
+  friend class fhandler_pty_slave;
 };
 
 class tty_list

winsup/utils/cygwin-console-helper.cc

@@ -1,12 +1,24 @@
 #include <windows.h>
+#include <stdio.h>
 int
 main (int argc, char **argv)
 {
   char *end;
-  if (argc != 3)
+  if (argc < 3)
     exit (1);
   HANDLE h = (HANDLE) strtoull (argv[1], &end, 0);
   SetEvent (h);
+  if (argc == 4) /* Pseudo console helper mode for PTY */
+    {
+      HANDLE hPipe = (HANDLE) strtoull (argv[3], &end, 0);
+      char buf[64];
+      sprintf (buf, "StdHandles=%p,%p\n",
+	       GetStdHandle (STD_INPUT_HANDLE),
+	       GetStdHandle (STD_OUTPUT_HANDLE));
+      DWORD dwLen;
+      WriteFile (hPipe, buf, strlen (buf), &dwLen, NULL);
+      CloseHandle (hPipe);
+    }
   h = (HANDLE) strtoull (argv[2], &end, 0);
   WaitForSingleObject (h, INFINITE);
   exit (0);