#!/bin/bash # Shell connector run background task and install functions for deal with. # (C) 2018-2023 Felix Hauri - felix@f-hauri.ch # Licensed under terms of GPL v3. www.gnu.org # 2020-09-08 use of `local -n`, `coproc` and `exec {FDNAM}<> >(: O)` # Version: 0.99.011 -- Last-Update: 2023-08-28 19:54:55 CEST # # Sourced, this script will define mostly 3 functions: # "newConnector" connect current bash with any tool like `date -f`, `bc`, ... # "newSqlConnector" connect with a DB client, like mysql, psql, sqlite... # "mySqlReq" to submit request with connect DB, as argument or STDIN # newConnector function will define function prepended by `my[CAP]` for # each backgrounded tasks. Sample: `newConnector /usr/bin/bc "-l"` will # define a myBc function to deal with backgrounded `bc`. # # Run as a script, without argument, this will run a demo, using `sqlite', # `bc`, `date`, run together, to show a recomputed ouptut of `/bin/df`, # store output of `/bin/ls` into a sqlite DB, then request DB for # output content on terminal. # Finally, show date returned by `sqlite`, `date` and `bash`, then # run `ps fw` on current session id, for showing running background # tasks, just before ending thems. # # shellcheck disable=SC2154 # function populated variables # SQLITE=/usr/bin/sqlite3 # Globals # Store a unique boundary string generated by the 'mkBound' function. # This boundary string is used to separate the output of different commands. declare bound # At each `sqlReq` invocation, add a SQL statment asking for current time and # titled by `$bound`. So when sqlReq find his `$bound`, next line to be read # is "current time". declare sqlreqbound # File descriptor associated with the input to the SQL client co-process. # It is used to send SQL commands to the SQL client. declare -i SQLIN # File descriptor associated with the output from the SQL client co-process. # It is used to read the responses of SQL commands from the SQL client. declare -i SQLOUT # File descriptor associated with the error stream of the SQL client co-process. # It is used to read any error messages that the SQL client might produce. declare -i SQLERR # Store current time answered at last line read by 'newSqlConnector'. # For debugging purposes. declare -i lastsqlread # List of back procs to kill before exit declare -ai tokill mkBound() { # mkBound() - Generate a unique boundary string # # This function generates a unique boundary string that is used to delimit # the output from sub-commands executed as co-processes, especially useful # for SQL responses with variable length. # # Usage: mkBound [varname] # [varname]: The name of the variable that will hold the generated string. # If not provided, defaults to a variable named "bound". # # The generated boundary string is a sequence of 30 random alphanumeric # characters and has the format: "--xxxxxxx-xxxx-xxxx-xxxx-xxxx-xxxx-", # where "x" stands for a random character. # # Constants: # BOUNDARY_LENGTH - The length of the boundary string to be generated. # BOUNDARY_ITERATIONS - The number of iterations in the loop # for generating the boundary string. # BOUNDARY_MASK - The mask used for selecting bits from the # pseudo-random numbers. # BOUNDARY_SHIFT - The number of bits to shift for the next # character in the boundary string. # # Notes: # - Builds some uniq 30 random char string from 12 $RANDOM values: # $RANDOM is 15 bits. 12 x 15 bits = 180 bits => 30 x 6 bits char # Constants for boundary construction local BOUNDARY_LENGTH=30 local BOUNDARY_ITERATIONS=6 local BOUNDARY_MASK=63 local BOUNDARY_SHIFT=6 # Named reference to output variable local -n result=${1:-bound} # The string used for generating the boundary local _Bash64_refstr _out='' _l _i _num printf -v _Bash64_refstr "%s" {0..9} {a..z} {A..Z} @ _ 0 # Generate boundary by looping BOUNDARY_ITERATIONS times # Construct BOUNDARY_LENGTH chars by shifting and masking RANDOM values # This is done by generating a pseudo-random number (_num) and then # selecting a character from _Bash64_refstr based on the bits in _num. # The bits are selected by shifting _num to the right by _i bits and then # applying the BOUNDARY_MASK to select the least significant 6 bits. # This is repeated BOUNDARY_LENGTH times to generate the boundary string. for ((_l=BOUNDARY_ITERATIONS;_num=(RANDOM<<15|RANDOM),_l--;));do for ((_i=0;_i>_i)&BOUNDARY_MASK:1} done done # Format the output into "--xxxxxxx-xxxx-xxxx-xxxx-xxxx-xxxx-" printf -v result -- "--%s-%s-%s-%s-%s-%s-" "${_out:0:7}" \ "${_out:7:4}" "${_out:11:4}" "${_out:15:4}" \ "${_out:19:4}" "${_out:23}"; } newConnector() { # newConnector() - Initiate a long-running subprocess for a given command # and associates it with two file descriptors. # # This function sets up a long-running co-process using the provided command # and arguments. It creates two file descriptors XXIN and XXOUT associated # with the co-process to manage data interactions, where 'XX' is the # uppercase form of the command name. It also dynamically creates a # function, named 'myXxx', to interact with the co-process. # 'Xxx' corresponds to the capitalized form of the command name. # # Synopsis: # newConnector "command" "command_args" "readiness_test" "expected_result" # newConnector "/path/to/command" \ # "-arg1 -arg2" "initialization input" "expected response" # # Parameters: # command - The command to be run as a co-process. # args - The arguments to be passed to the command. # readiness_test - The initial input to be sent to the co-process # for readiness verification. # expexted_result - The expected output from the command when # 'readiness_test' is sent as input. # # Returns: # None directly. However, it prints a warning message to STDERR if # the verification of the co-process fails. # # Constants: # cinfd - nameref File descriptor for input to the co-process (XXIN). # coutfd - nameref File descriptor for output from the co-process (XXOUT). # # Notes: # - The function 'myXxx' sends input to the command and reads the output # into the 'result' variable (nameref, default to xxOut). # - The function 'myXxx' prints the result to STDOUT if there is no # second argument (also stored into xxOut variable). # - If sending the 'readiness_test' does not yield the 'expected_result' # string, a warning is printed to STDERR. local command="$1" cmd=${1##*/} args \ readiness_test="$3" expected_result="$4" timeout=3 initfile input read -ra args <<<"$2" shift 4 local -n cinfd=${cmd^^}IN coutfd=${cmd^^}OUT # Start a new co-process using the provided command and arguments # The output of the command is unbuffered (-o0 option to stdbuf) # This co-process can interact with the main process via its standard # input and output ## Workaround 'warning: execute_coproc: coproc [xx:COPROC] still exists' ### coproc stdbuf -o0 "$command" "$args" 2>&1 ### cinfd=${COPROC[1]} coutfd=${COPROC[0]} exec {cinfd}<> <(: ┋) # shellcheck disable=SC2034 # CommandOutFd is used externally exec {coutfd}< <(stdbuf -o0 "$command" "${args[@]}" 2>&1 <&"${cinfd}") tokill+=($!) # Feed the initialization file to the command for initfile; do cat >&"${cinfd}" "$initfile" done # Dynamically create a function that sends input to the backgrounded command # and reads its output into a variable. # The function name is 'my' followed by the capitalized command name # (e.g., myBc, myDate, myUconv ...) # The function takes one argument, sends it to the command, and reads the # response into the 'result' variable # shellcheck disable=SC1091 source /dev/stdin <<-EOF my${cmd^}() { local -n result=\${2:-${cmd}Out} # Nameref to the output variable # Send input to the command echo >&\${${cmd^^}IN} "\$1" && # Read the response with a timeout of $timeout seconds read -u \${${cmd^^}OUT} -t "$timeout" result # If there is no second argument, print the result ((\$#>1)) || echo \$result } EOF # Check the command by sending the 'readiness_test' and comparing the # response to 'expected_result' my"${cmd^}" "$readiness_test" input if [[ $input != "$expected_result" ]]; then printf >&2 "WARNING: Don't match! '%s' <> '%s'.\n" \ "$expected_result" "$input" fi } newSqlConnector() { # newSqlConnector() - Establish a connection to a specified SQL client # # Establishes a long-running connection to an SQL client and prepares the # environment necessary for executing SQL queries on that client. # It currently supports "SQLite", "MySQL", "MariaDB", and "PostgreSQL". # Field separator must be tabulation $'\t'! # The output length is not fixed and could be empty. # # Synopsis: # newSqlConnector sqlite3 $'-separator \t -header /path/to/dbfile.sqlite' # newSqlConnector psql $'-Anh hostname -F \t --pset=footer=off user' # newSqlConnector mysql '-h hostname -B -p database' # # Parameters: # $1 (command) - The command to execute (i.e., the SQL client) # $2 (args) - The command-line arguments for the SQL client, required # for host, user and database selection, and also set # as field separator. # # Returns: # None directly. But it sets up SQL input and output file descriptors # for subsequent interaction with the SQL client. # # Constants: # SQLIN - A file descriptor for sending inputs to the SQL client. # SQLOUT - A file descriptor for reading the SQL client's outputs. # SQLERR - A file descriptor for the SQL client's error stream # # Notes: # - The function assumes that the command passed to it is an executable, # so it should be validated before calling this function. # - If an unknown SQL client is passed, the function will print a warning # but will not terminate! # Take the SQL client command and command arguments as input local command="$1" cmd=${1##*/} args COPROC # Split the command arguments IFS=' ' read -r -a args <<<"$2" # Generate a unique boundary string mkBound bound # Determine the SQL client and set `sqlreqbound` for each client case $cmd in psql ) sqlreqbound='EXTRACT(\047EPOCH\047 FROM now())' ;; sqlite* ) sqlreqbound='STRFTIME(\047%%s\047,DATETIME(\047now\047))' ;; mysql|mariadb ) sqlreqbound='UNIX_TIMESTAMP()' ;; * ) # If the SQL client is not recognized, give a warning echo >&2 "WARNING '$cmd' not known as SQL client";; esac # Create a new file descriptor for SQL errors ( ``p'' is tonge's smiley ) exec {SQLERR}<> <(: p) # Start the SQL client as a co-process, with its standard error redirected # to `SQLERR` coproc stdbuf -o0 "$command" "${args[@]}" 2>&"$SQLERR" tokill+=($!) # Store the file descriptors for the co-process's standard input and output SQLIN=${COPROC[1]} SQLOUT=${COPROC[0]} } mySqlReq() { # mySqlReq() - Send SQL commands to the SQL client and manages responses # # This function takes the name of a variable as the first argument. It is # designed to send an SQL command to the SQL client, read response, # and store response into the variable specified by the first argument. # The function doesn't return any value but populates three variables based # on the argument: `$1`, `${1}_h`, and `${1}_e`. The variables: # - `$1` is an array containing the SQL response's datas, # - `${1}_h` is an array containing the header fields, and # - `${1}_e` is an array containing each lines of errors, if any occur. # # The SQL command to be executed can be passed in two ways: # 1. Directly as the second argument in the function call: # mySqlReq result "SELECT * FROM table_name" # 2. Piped into from standard input if no second argument is provided # mySqlReq result <<<"SELECT * FROM table_name" # When only the variable name is provided as an argument, the function # waits for the SQL command from standard input. # In both cases, the result of the SQL command execution is stored in the # variables as described (${result[@]}, ${result_h[@]} and ${result_e[@]}). # # Synopsis: # mySqlReq result "SELECT * FROM table_name" # mySqlReq result SELECT '*' FROM "table_name" # mySqlReq result <&"$SQLIN" '%s;\n' "${*%;}" else local -a _req mapfile -t _req printf >&"$SQLIN" '%s;\n' "${_req[*]%;}" fi # Request the output of the unique boundary string which was defined in # the `newSqlConnector` function. printf >&"$SQLIN" 'SELECT '"${sqlreqbound}"' AS "%s";\n' "$bound" # Read the response from the SQL client co-process. read -ru $SQLOUT line # should be header line if [[ $line != "$bound" ]]; then # If boundary, then no datas. # shellcheck disable=SC2034 # Result Header is used externally IFS=$'\t' read -ra result_h <<< "$line" while read -ru $SQLOUT line && [[ $line != "$bound" ]]; do result+=("$line") done fi read -ru $SQLOUT line lastsqlread="$line" if read -u $SQLERR -t 0; then while read -ru "$SQLERR" -t "$timeout" line; do result_e+=("$line") done fi } beforExit() { kill "${tokill[@]}" } trap beforExit 0 1 2 3 6 15 ### ### End of ``connectors'' definitions ### # # Exit here if script is sourced [[ $0 = "${BASH_SOURCE[0]}" ]] || { true return 0 } ### ### Demo-sample: define/open 3 connectors to `bc`, `date` and `sqlite`, ### for interactive use from main bash script. ### # Test if sqlite is present if [[ ! -x $SQLITE ]]; then echo >&2 "Tool '$SQLITE' not found (please correct \$SQLITE if installed)." exit 1 fi # 1. instanciate a long-running bc command which we will then be # able to interact with with myBc(). # Initialize them by declaring `mil` function, to compute # human readable, returning `A,XX.YYY` where A mean power of 1K # and XX.YYY is value divided by 1024**A. newConnector /usr/bin/bc '-l' 'mil(0)' '0,0' - <<-"EoBcInit" define void mil (s) { if (s==0) { print "0,0\n"; return;}; p=l(s)/l(1024); scale=0; p=p/1; scale=20; print p,",",s/1024^p,"\n"; } EoBcInit # 2. instanciate a long-running date command which we will then be able # to interact with with myDate(), convert date input to UnixTimeStamp. newConnector /bin/date '-f - +%s' @0 0 # 3. instanciate a long-running sqlite command with file based on /dev/shm # Interact with mySqlReq newSqlConnector $SQLITE $'-separator \t -header /run/user'/"$UID/test.sqlite" # Database initialisation... cat >&$SQLIN <<-EOSQLInit DROP TABLE IF EXISTS files; CREATE TABLE files (perms, user, date UNSIGNED BIGINT, size UNSIGNED BIGINT,name); EOSQLInit declare -a ABR=(K M G T P) # First: A simple demo show `df` output, using backgrounded `bc -l` (No DB). { read -r _ # Ignore df's headline echo "dtot=0;duse=0;" >&"$BCIN" while read -r _ type size used free prct mpoint; do echo >&"$BCIN" "dtot+=$used+$free;duse+=$used;" myBc "100*$used/($used+$free)" mpct # Compute % of use myBc "mil($used)" Used # Human readable form myBc "mil($used+$free)" Total printf "%-32s %-22s %7.2f%s %8.2f%s %9s %8.2f%%\n" "$mpoint" "$type" \ "${Used#*,}" "${ABR[${Used%,*}]}" "${Total#*,}" \ "${ABR[${Total%,*}]}" "$prct" "$mpct" done } < <(LANG=C df -kT) myBc "mil(dtot)" Total myBc "mil(duse)" Used printf "%-36s%7.2f%s %7.2f%s\n\n" Total \ "${Used#*,}" "${ABR[${Used%,*}]}" "${Total#*,}" "${ABR[${Total%,*}]}" # Prepare to work agains files ABR=('' "${ABR[@]}") # lowest is byte, not kb. echo "ftot=0;" >&"$BCIN" # file size total = 0, but as BC variable # Let play with SQL putting all current directory into a table. { read -r _ # Ignore ls's 1st line while read -r perm _ user _ size month day yot name; do echo >&"$BCIN" "ftot+=$size" # Increment BC variable ftot. myDate "$month $day $yot" Date # Convert `ls` date format to EPOCH printf >&"$SQLIN" "INSERT INTO files values('%s','%s',%s,%s,'%s');" \ "$perm" "$user" "$Date" "$size" "$name" # DB INSERT done } < <(LANG=C /bin/ls --full-time -Al) mySqlReq myarray "SELECT * from files order by date;" # DB SELECT files by date printf "Current working dir, from database\n%-12s %-12s %16s %10s %s\n" \ "${myarray_h[@]}" # print header for line in "${myarray[@]}"; do IFS=$'\t' read -r perm user date size name <<<"$line" myBc "mil($size)" Size printf " %-11s %-12s %(%F %H:%M)T %7.2f%-2s %s\n" "$perm" "$user" \ "$date" "${Size#*,}" "${ABR[${Size%,*}]}b" "$name" done myBc "mil(ftot)" Total # Total computed by bc mySqlReq stot 'SELECT SUM(size) FROM files;' # DB SELECT SUM of file size myBc "mil($stot)" STotal # Total computed by DB printf "%18sby bc:%7.2f%-2s, by sql:%7.2f%-2s %s\n" '' "${Total#*,}" \ "${ABR[${Total%,*}]}b" "${STotal#*,}" "${ABR[${STotal%,*}]}b" Total myBc ftot bctot # bc variable ftot to bash var bctot (hope BC and SQL match!) [[ $bctot == "$stot" ]] || echo "WARN: BC:'$bctot' and SQL:'$stot' don't match!" myDate now now # Date from backgrounded date task printf "%-14s: %(%a %d %b %T)T\n" \ "Last SQL read" "$lastsqlread" "now by date" "$now" "now by bash" -1 exec {BCIN}>&- exec {DATEIN}>&- read -ru "$BCOUT" -t .01 _ read -ru "$DATEOUT" -t .01 _ exec {BCOUT}<&- exec {DATEOUT}<&- mySid=$(ps ho sid $$) ps --sid "$mySid" fw