sandbox: Add a function to detect terminal connection

Add a serial_is_tty() function that determines if the serial console
is connected to a terminal. For sandbox, this uses os_isatty() to
check stdin, except for cooked mode, where we don't want to assume
anything about the terminal.

For other platforms, it always returns true.

Co-developed-by: Claude <noreply@anthropic.com>
Signed-off-by: Simon Glass <sjg@chromium.org>

arch/sandbox/cpu/os.c

@@ -142,6 +142,11 @@ int os_close(int fd)
 	return -1;
 }
 
+int os_isatty(int fd)
+{
+	return isatty(fd);
+}
+
 int os_unlink(const char *pathname)
 {
 	return unlink(pathname);

arch/sandbox/cpu/start.c

@@ -613,6 +613,10 @@ int sandbox_main(int argc, char *argv[])
 	if (os_parse_args(state, argc, argv))
 		return 1;
 
+	/* Detect if serial console is connected to a terminal */
+	state->serial_is_tty = os_isatty(1) &&
+		state->term_raw != STATE_TERM_COOKED;
+
 	if (state->ram_buf_fname) {
 		ret = os_read_ram_buf(state->ram_buf_fname);
 		if (ret) {

arch/sandbox/cpu/state.c

@@ -476,6 +476,13 @@ bool sandbox_sf_bootdev_enabled(void)
 	return !state->disable_sf_bootdevs;
 }
 
+bool sandbox_serial_is_tty(void)
+{
+	struct sandbox_state *state = state_get_current();
+
+	return state->serial_is_tty;
+}
+
 int state_init(void)
 {
 	state = &main_state;

arch/sandbox/include/asm/state.h

@@ -157,6 +157,7 @@ struct sandbox_state {
 	bool ignore_missing_state_on_read;	/* No error if state missing */
 	bool show_lcd;			/* Show LCD on start-up */
 	bool double_lcd;		/* Double display size for high-DPI */
+	bool serial_is_tty;		/* Serial console is connected to a tty */
 	enum sysreset_t last_sysreset;	/* Last system reset type */
 	bool sysreset_allowed[SYSRESET_COUNT];	/* Allowed system reset types */
 	enum state_terminal_raw term_raw;	/* Terminal raw/cooked */
@@ -377,6 +378,13 @@ int state_get_rel_filename(const char *rel_path, char *buf, int size);
  */
 int state_load_other_fdt(const char **bufp, int *sizep);
 
+/**
+ * sandbox_serial_is_tty() - check if serial console is connected to a tty
+ *
+ * Return: true if serial console is connected to a terminal, false if not
+ */
+bool sandbox_serial_is_tty(void);
+
 /**
  * Initialize the test system state
  */

include/os.h

@@ -90,6 +90,14 @@ int os_open(const char *pathname, int flags);
  */
 int os_close(int fd);
 
+/**
+ * os_isatty() - check if file descriptor refers to a terminal
+ *
+ * @fd:		File descriptor to check
+ * Return:	1 if fd is a terminal, 0 if not, -1 on error
+ */
+int os_isatty(int fd);
+
 /**
  * os_unlink() - access to the OS unlink() system call
  *

include/serial.h

@@ -2,6 +2,9 @@
 #define __SERIAL_H__
 
 #include <post.h>
+#ifdef CONFIG_SANDBOX
+#include <asm/state.h>
+#endif
 
 /* Escape value */
 #define cESC	'\x1b'
@@ -423,4 +426,22 @@ int serial_query_size(int *rowsp, int *colsp);
  */
 int serial_get_size(int *rowsp, int *colsp);
 
+/*
+ * serial_is_tty() - check if the serial console is connected to a terminal
+ *
+ * This does not indicate that there is actually a terminal, only that if there
+ * is one, we can assume it is present and connected
+ *
+ * Return: true if any serial console is likely connected to a terminal, false if not
+ */
+static inline bool serial_is_tty(void)
+{
+#ifdef CONFIG_SANDBOX
+	return sandbox_serial_is_tty();
+#else
+	/* assume that it is! */
+	return true;
+#endif
+}
+
 #endif