diff options
Diffstat (limited to 'testing/sgh_dd.cpp')
| -rw-r--r-- | testing/sgh_dd.cpp | 5090 |
1 files changed, 5090 insertions, 0 deletions
diff --git a/testing/sgh_dd.cpp b/testing/sgh_dd.cpp new file mode 100644 index 00000000..b0704ffa --- /dev/null +++ b/testing/sgh_dd.cpp @@ -0,0 +1,5090 @@ +/* + * A utility program for copying files. Specialised for "files" that + * represent devices that understand the SCSI command set. + * + * Copyright (C) 2018-2022 D. Gilbert + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This program is a specialisation of the Unix "dd" command in which + * one or both of the given files is a scsi generic device. + * A logical block size ('bs') is assumed to be 512 if not given. This + * program complains if 'ibs' or 'obs' are given with some other value + * than 'bs'. If 'if' is not given or 'if=-' then stdin is assumed. If + * 'of' is not given or 'of=-' then stdout assumed. + * + * A non-standard argument "bpt" (blocks per transfer) is added to control + * the maximum number of blocks in each transfer. The default value is 128. + * For example if "bs=512" and "bpt=32" then a maximum of 32 blocks (16 KiB + * in this case) are transferred to or from the sg device in a single SCSI + * command. + * + * This version is designed for the linux kernel 2.4, 2.6, 3, 4 and 5 series. + * + * sgp_dd is a Posix threads specialization of the sg_dd utility. Both + * sgp_dd and sg_dd only perform special tasks when one or both of the given + * devices belong to the Linux sg driver. + * + * sgh_dd further extends sgp_dd to use the experimental kernel buffer + * sharing feature added in 3.9.02 . + * N.B. This utility was previously called sgs_dd but there was already an + * archived version of a dd variant called sgs_dd so this utility name was + * renamed [20181221] + */ + +static const char * version_str = "2.22 20221020"; + +#define _XOPEN_SOURCE 600 +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif + +#include <unistd.h> +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> +#include <stdbool.h> +#include <string.h> +#include <ctype.h> +#include <errno.h> +#include <poll.h> +#include <limits.h> +#include <pthread.h> +#include <signal.h> +#define __STDC_FORMAT_MACROS 1 +#include <inttypes.h> +#include <sys/ioctl.h> +#include <sys/stat.h> +#include <sys/sysmacros.h> +#ifndef major +#include <sys/types.h> +#endif +#include <sys/time.h> +#include <linux/major.h> /* for MEM_MAJOR, SCSI_GENERIC_MAJOR, etc */ +#include <linux/fs.h> /* for BLKSSZGET and friends */ +#include <sys/mman.h> /* for mmap() system call */ + +#include <vector> +#include <array> +#include <atomic> // C++ header replacing <stdatomic.h> also link + // needed '-l atomic' . Not anymore?? +#include <random> +#include <thread> // needed for std::this_thread::yield() +#include <mutex> +#include <chrono> + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_GETRANDOM +#include <sys/random.h> /* for getrandom() system call */ +#endif + +#ifndef HAVE_LINUX_SG_V4_HDR +/* Kernel uapi header contain __user decorations on user space pointers + * to indicate they are unsafe in the kernel space. However glibc takes + * all those __user decorations out from headers in /usr/include/linux . + * So to stop compile errors when directly importing include/uapi/scsi/sg.h + * undef __user before doing that include. */ +#define __user + +/* Want to block the original sg.h header from also being included. That + * causes lots of multiple definition errors. This will only work if this + * header is included _before_ the original sg.h header. */ +#define _SCSI_GENERIC_H /* original kernel header guard */ +#define _SCSI_SG_H /* glibc header guard */ + +#include "uapi_sg.h" /* local copy of include/uapi/scsi/sg.h */ + +#else +#define __user +#endif /* end of: ifndef HAVE_LINUX_SG_V4_HDR */ + +#include "sg_lib.h" +#include "sg_cmds_basic.h" +#include "sg_io_linux.h" +#include "sg_unaligned.h" +#include "sg_pr2serr.h" + + +using namespace std; + +#ifdef __GNUC__ +#ifndef __clang__ +#pragma GCC diagnostic ignored "-Wclobbered" +#endif +#endif + +/* comment out following line to stop ioctl(SG_CTL_FLAGM_SNAP_DEV) */ +#define SGH_DD_SNAP_DEV 1 + +#ifndef SGV4_FLAG_POLLED +#define SGV4_FLAG_POLLED 0x800 +#endif + +#define DEF_BLOCK_SIZE 512 +#define DEF_BLOCKS_PER_TRANSFER 128 +#define DEF_BLOCKS_PER_2048TRANSFER 32 +#define DEF_SDT_ICT_MS 300 +#define DEF_SDT_CRT_SEC 3 +#define DEF_SCSI_CDBSZ 10 +#define MAX_SCSI_CDBSZ 16 +#define MAX_BPT_VALUE (1 << 24) /* used for maximum bs as well */ +#define MAX_COUNT_SKIP_SEEK (1LL << 48) /* coverity wants upper bound */ + +#define SENSE_BUFF_LEN 64 /* Arbitrary, could be larger */ +#define READ_CAP_REPLY_LEN 8 +#define RCAP16_REPLY_LEN 32 + +#define DEF_TIMEOUT 60000 /* 60,000 millisecs == 60 seconds */ + +#define SGP_READ10 0x28 +#define SGP_PRE_FETCH10 0x34 +#define SGP_PRE_FETCH16 0x90 +#define SGP_VERIFY10 0x2f +#define SGP_WRITE10 0x2a +#define DEF_NUM_THREADS 4 +#define MAX_NUM_THREADS 1024 /* was SG_MAX_QUEUE with v3 driver */ +#define DEF_NUM_MRQS 0 + +#define FT_OTHER 1 /* filetype other than one of the following */ +#define FT_SG 2 /* filetype is sg char device */ +#define FT_DEV_NULL 4 /* either /dev/null, /dev/zero or "." */ +#define FT_ST 8 /* filetype is st char device (tape) */ +#define FT_CHAR 16 /* filetype is st char device (tape) */ +#define FT_BLOCK 32 /* filetype is a block device */ +#define FT_FIFO 64 /* fifo (named or unnamed pipe (stdout)) */ +#define FT_RANDOM_0_FF 128 /* iflag=00, iflag=ff and iflag=random + override if=IFILE */ +#define FT_ERROR 256 /* couldn't "stat" file */ + +#define DEV_NULL_MINOR_NUM 3 +#define DEV_ZERO_MINOR_NUM 5 + +#define EBUFF_SZ 768 + +#define PROC_SCSI_SG_VERSION "/proc/scsi/sg/version" +#define SYS_SCSI_SG_VERSION "/sys/module/sg/version" + +struct flags_t { + bool append; + bool coe; + bool defres; /* without this res_sz==bs*bpt */ + bool dio; + bool direct; + bool dpo; + bool dsync; + bool excl; + bool ff; + bool fua; + bool polled; /* formerly called 'hipri' */ + bool masync; /* more async sg v4 driver flag */ + bool mrq_immed; /* mrq submit non-blocking */ + bool mrq_svb; /* mrq shared_variable_block, for sg->sg copy */ + bool no_dur; + bool nocreat; + bool noshare; + bool no_thresh; + bool no_unshare; /* leave it for driver close/release */ + bool no_waitq; /* dummy, no longer supported, just warn */ + bool noxfer; + bool qhead; + bool qtail; + bool random; + bool mout_if; /* META_OUT_IF flag at mrq level */ + bool same_fds; + bool swait; /* now ignore; kept for backward compatibility */ + bool v3; + bool v4; + bool v4_given; + bool wq_excl; + bool zero; + int mmap; +}; + +struct global_collection +{ /* one instance visible to all threads */ + int infd; + int64_t skip; + int in_type; + int cdbsz_in; + int help; + int elem_sz; + struct flags_t in_flags; + // int64_t in_blk; /* -\ next block address to read */ + // int64_t in_count; /* | blocks remaining for next read */ + atomic<int64_t> in_rem_count; /* | count of remaining in blocks */ + atomic<int> in_partial; /* | */ + atomic<bool> in_stop; /* | */ + off_t in_st_size; /* Only for FT_OTHER (regular) file */ + pthread_mutex_t in_mutex; /* -/ */ + int nmrqs; /* Number of multi-reqs for sg v4 */ + int outfd; + int64_t seek; + int out_type; + int out2fd; + int out2_type; + int cdbsz_out; + int aen; /* abort every nth command */ + int m_aen; /* abort mrq every nth command */ + struct flags_t out_flags; + atomic<int64_t> out_blk; /* -\ next block address to write */ + atomic<int64_t> out_count; /* | blocks remaining for next write */ + atomic<int64_t> out_rem_count; /* | count of remaining out blocks */ + atomic<int> out_partial; /* | */ + atomic<bool> out_stop; /* | */ + off_t out_st_size; /* Only for FT_OTHER (regular) file */ + pthread_mutex_t out_mutex; /* | */ + pthread_cond_t out_sync_cv; /* | hold writes until "in order" */ + pthread_mutex_t out2_mutex; + int bs; + int bpt; + int cmd_timeout; /* in milliseconds */ + int outregfd; + int outreg_type; + int ofsplit; + atomic<int> dio_incomplete_count; + atomic<int> sum_of_resids; + uint32_t sdt_ict; /* stall detection; initial check time (milliseconds) */ + uint32_t sdt_crt; /* check repetition time (seconds), after first stall */ + int fail_mask; + int verbose; + int dry_run; + int chkaddr; + bool aen_given; + bool cdbsz_given; + bool is_mrq_i; + bool is_mrq_o; + bool m_aen_given; + bool ofile_given; + bool ofile2_given; + bool unit_nanosec; /* default duration unit is millisecond */ + bool mrq_cmds; /* mrq=<NRQS>,C given */ + bool mrq_async; /* mrq_immed flag given */ + bool noshare; /* don't use request sharing */ + bool unbalanced_mrq; /* so _not_ sg->sg request sharing sync mrq */ + bool verify; /* don't copy, verify like Unix: cmp */ + bool prefetch; /* for verify: do PF(b),RD(a),V(b)_a_data */ + bool unshare; /* let close() do file unshare operation */ + const char * infp; + const char * outfp; + const char * out2fp; +}; + +typedef struct mrq_abort_info +{ + int from_tid; + int fd; + int mrq_id; + int debug; +} Mrq_abort_info; + +typedef struct request_element +{ /* one instance per worker thread */ + struct global_collection *clp; + bool wr; + bool has_share; + bool both_sg; + bool same_sg; + bool only_in_sg; + bool only_out_sg; + // bool mrq_abort_thread_active; + int id; + int bs; + int infd; + int outfd; + int out2fd; + int outregfd; + int64_t iblk; + int64_t oblk; + int num_blks; + uint8_t * buffp; + uint8_t * alloc_bp; + struct sg_io_hdr io_hdr; + struct sg_io_v4 io_hdr4[2]; + uint8_t cmd[MAX_SCSI_CDBSZ]; + uint8_t sb[SENSE_BUFF_LEN]; + int dio_incomplete_count; + int mmap_active; + int resid; + int rd_p_id; + int rep_count; + int rq_id; + int mmap_len; + int mrq_id; + int mrq_index; + uint32_t in_mrq_q_blks; + uint32_t out_mrq_q_blks; + long seed; +#ifdef HAVE_SRAND48_R /* gcc extension. N.B. non-reentrant version slower */ + struct drand48_data drand;/* opaque, used by srand48_r and mrand48_r */ +#endif + pthread_t mrq_abort_thread_id; + Mrq_abort_info mai; +} Rq_elem; + +typedef struct thread_info +{ + int id; + struct global_collection * gcp; + pthread_t a_pthr; +} Thread_info; + +/* Additional parameters for sg_start_io() and sg_finish_io() */ +struct sg_io_extra { + bool is_wr2; + bool prefetch; + bool dout_is_split; + int hpv4_ind; + int blk_offset; + int blks; +}; + +#define MONO_MRQ_ID_INIT 0x10000 + +// typedef vector< pair<int, struct sg_io_v4> > mrq_arr_t; +typedef array<uint8_t, 32> big_cdb; /* allow up to a 32 byte cdb */ +typedef pair< vector<struct sg_io_v4>, vector<big_cdb> > mrq_arr_t; + + +/* Use this class to wrap C++11 <random> features to produce uniform random + * unsigned ints in the range [lo, hi] (inclusive) given a_seed */ +class Rand_uint { +public: + Rand_uint(unsigned int lo, unsigned int hi, unsigned int a_seed) + : uid(lo, hi), dre(a_seed) { } + /* uid ctor takes inclusive range when integral type */ + + unsigned int get() { return uid(dre); } + +private: + uniform_int_distribution<unsigned int> uid; + default_random_engine dre; +}; + +static atomic<int> mono_pack_id(1); +static atomic<int> mono_mrq_id(MONO_MRQ_ID_INIT); +static atomic<long int> pos_index(0); + +static atomic<int> num_ebusy(0); +static atomic<int> num_start_eagain(0); +static atomic<int> num_fin_eagain(0); +static atomic<int> num_abort_req(0); +static atomic<int> num_abort_req_success(0); +static atomic<int> num_mrq_abort_req(0); +static atomic<int> num_mrq_abort_req_success(0); +static atomic<int> num_miscompare(0); +static atomic<long> num_waiting_calls(0); +static atomic<bool> vb_first_time(true); +static atomic<bool> shutting_down(false); + +static sigset_t signal_set; +static sigset_t orig_signal_set; +static pthread_t sig_listen_thread_id; + +static const char * sg_allow_dio = "/sys/module/sg/parameters/allow_dio"; + +static void sg_in_rd_cmd(struct global_collection * clp, Rq_elem * rep, + mrq_arr_t & def_arr); +static void sg_out_wr_cmd(Rq_elem * rep, mrq_arr_t & def_arr, bool is_wr2, + bool prefetch); +static bool normal_in_rd(Rq_elem * rep, int blocks); +static void normal_out_wr(Rq_elem * rep, int blocks); +static int sg_start_io(Rq_elem * rep, mrq_arr_t & def_arr, int & pack_id, + struct sg_io_extra *xtrp); +static int sg_finish_io(bool wr, Rq_elem * rep, int pack_id, + struct sg_io_extra *xtrp); +static int sg_in_open(struct global_collection *clp, const char *inf, + uint8_t **mmpp, int *mmap_len); +static int sg_out_open(struct global_collection *clp, const char *outf, + uint8_t **mmpp, int *mmap_len); +static int sgh_do_deferred_mrq(Rq_elem * rep, mrq_arr_t & def_arr); + +#define STRERR_BUFF_LEN 128 + +static pthread_mutex_t strerr_mut = PTHREAD_MUTEX_INITIALIZER; + +static bool have_sg_version = false; +static int sg_version = 0; +static bool sg_version_lt_4 = false; +static bool sg_version_ge_40045 = false; +static bool do_sync = false; +static int do_time = 1; +static struct global_collection gcoll; +static struct timeval start_tm; +static int64_t dd_count = -1; +static int num_threads = DEF_NUM_THREADS; +static int exit_status = 0; +static bool after1 = false; + +static const char * my_name = "sgh_dd: "; + +static const char * mrq_blk_s = "mrq: ordinary blocking"; +static const char * mrq_vb_s = "mrq: variable blocking"; +static const char * mrq_svb_s = "mrq: shared variable blocking (svb)"; +static const char * mrq_s_nb_s = "mrq: submit of full non-blocking"; + + +#ifdef __GNUC__ +static int pr2serr_lk(const char * fmt, ...) + __attribute__ ((format (printf, 1, 2))); +#if 0 +static void pr_errno_lk(int e_no, const char * fmt, ...) + __attribute__ ((format (printf, 2, 3))); +#endif +#else +static int pr2serr_lk(const char * fmt, ...); +#if 0 +static void pr_errno_lk(int e_no, const char * fmt, ...); +#endif +#endif + + +static int +pr2serr_lk(const char * fmt, ...) +{ + int n; + va_list args; + + pthread_mutex_lock(&strerr_mut); + va_start(args, fmt); + n = vfprintf(stderr, fmt, args); + va_end(args); + pthread_mutex_unlock(&strerr_mut); + return n; +} + +static void +usage(int pg_num) +{ + if (pg_num > 3) + goto page4; + else if (pg_num > 2) + goto page3; + else if (pg_num > 1) + goto page2; + + pr2serr("Usage: sgh_dd [bs=BS] [conv=CONVS] [count=COUNT] [ibs=BS] " + "[if=IFILE]\n" + " [iflag=FLAGS] [obs=BS] [of=OFILE] [oflag=FLAGS] " + "[seek=SEEK]\n" + " [skip=SKIP] [--help] [--version]\n\n"); + pr2serr(" [ae=AEN[,MAEN]] [bpt=BPT] [cdbsz=6|10|12|16] " + "[coe=0|1]\n" + " [dio=0|1] [elemsz_kb=EKB] [fail_mask=FM] " + "[fua=0|1|2|3]\n" + " [mrq=[I|O,]NRQS[,C]] [noshare=0|1] " + "[of2=OFILE2]\n" + " [ofreg=OFREG] [ofsplit=OSP] [sdt=SDT] " + "[sync=0|1]\n" + " [thr=THR] [time=0|1|2[,TO]] [unshare=1|0] " + "[verbose=VERB]\n" + " [--dry-run] [--prefetch] [-v|-vv|-vvv] " + "[--verbose]\n" + " [--verify] [--version]\n\n" + " where the main options (shown in first group above) are:\n" + " bs must be device logical block size (default " + "512)\n" + " conv comma separated list from: [nocreat,noerror," + "notrunc,\n" + " null,sync]\n" + " count number of blocks to copy (def: device size)\n" + " if file or device to read from (def: stdin)\n" + " iflag comma separated list from: [00,coe,defres,dio," + "direct,dpo,\n" + " dsync,excl,ff,fua,masync,mmap,mout_if," + "mrq_immed,mrq_svb,\n" + " nocreat,nodur,noxfer,null,polled,qhead," + "qtail,\n" + " random,same_fds,v3,v4,wq_excl]\n" + " of file or device to write to (def: /dev/null " + "N.B. different\n" + " from dd it defaults to stdout). If 'of=.' " + "uses /dev/null\n" + " of2 second file or device to write to (def: " + "/dev/null)\n" + " oflag comma separated list from: [append,<<list from " + "iflag>>]\n" + " seek block position to start writing to OFILE\n" + " skip block position to start reading from IFILE\n" + " --help|-h output this usage message then exit\n" + " --verify|-x do a verify (compare) operation [def: do a " + "copy]\n" + " --version|-V output version string then exit\n\n" + "Copy IFILE to OFILE, similar to dd command. This utility is " + "specialized for\nSCSI devices and uses multiple POSIX threads. " + "It expects one or both IFILE\nand OFILE to be sg devices. With " + "--verify option does a verify/compare\noperation instead of a " + "copy. This utility is Linux specific and uses the\nv4 sg " + "driver 'share' capability if available. Use '-hh', '-hhh' or " + "'-hhhh'\nfor more information.\n" + ); + return; +page2: + pr2serr("Syntax: sgh_dd [operands] [options]\n\n" + " where: operands have the form name=value and are pecular to " + "'dd'\n" + " style commands, and options start with one or " + "two hyphens;\n" + " the lesser used operands and option are:\n\n" + " ae AEN: abort every n commands (def: 0 --> don't " + "abort any)\n" + " MAEN: abort every n mrq commands (def: 0 --> " + "don't)\n" + " [requires commands with > 1 ms duration]\n" + " bpt is blocks_per_transfer (default is 128)\n" + " cdbsz size of SCSI READ, WRITE or VERIFY cdb_s " + "(default is 10)\n" + " coe continue on error, 0->exit (def), " + "1->zero + continue\n" + " dio is direct IO, 1->attempt, 0->indirect IO (def)\n" + " elemsz_kb scatter gather list element size in kilobytes " + "(def: 32[KB])\n" + " fail_mask 1: misuse KEEP_SHARE flag; 0: nothing (def)\n" + " fua force unit access: 0->don't(def), 1->OFILE, " + "2->IFILE,\n" + " 3->OFILE+IFILE\n" + " mrq number of cmds placed in each sg call " + "(def: 0);\n" + " may have trailing ',C', to send bulk cdb_s; " + "if preceded\n" + " by 'I' then mrq only on IFILE, likewise 'O' " + "for OFILE\n" + " noshare 0->use request sharing(def), 1->don't\n" + " ofreg OFREG is regular file or pipe to send what is " + "read from\n" + " IFILE in the first half of each shared element\n" + " ofsplit split ofile write in two at block OSP (def: 0 " + "(no split))\n" + " sdt stall detection times: CRT[,ICT]. CRT: check " + "repetition\n" + " time (after first) in seconds; ICT: initial " + "check time\n" + " in milliseconds. Default: 3,300 . Use CRT=0 " + "to disable\n" + " sync 0->no sync(def), 1->SYNCHRONIZE CACHE on OFILE " + "after copy\n" + " thr is number of threads, must be > 0, default 4, " + "max 1024\n" + " time 0->no timing, 1->calc throughput(def), " + "2->nanosec\n" + " precision; TO is command timeout in seconds " + "(def: 60)\n" + " unshare 0->don't explicitly unshare after share; 1->let " + "close do\n" + " file unshare (default)\n" + " verbose increase verbosity\n" + " --chkaddr|-c exits if read block does not contain " + "32 bit block\n" + " address, used once only checks first " + "address in block\n" + " --dry-run|-d prepare but bypass copy/read\n" + " --prefetch|-p with verify: do pre-fetch first\n" + " --verbose|-v increase verbosity of utility\n\n" + "Use '-hhh' or '-hhhh' for more information about flags.\n" + ); + return; +page3: + pr2serr("Syntax: sgh_dd [operands] [options]\n\n" + " where: 'iflag=<arg>' and 'oflag=<arg>' arguments are listed " + "below:\n\n" + " 00 use all zeros instead of if=IFILE (only in " + "iflags)\n" + " 00,ff generates blocks that contain own (32 bit be) " + "blk address\n" + " append append output to OFILE (assumes OFILE is " + "regular file)\n" + " coe continue of error (reading, fills with zeros)\n" + " defres keep default reserve buffer size (else its " + "bs*bpt)\n" + " dio sets the SG_FLAG_DIRECT_IO in sg requests\n" + " direct sets the O_DIRECT flag on open()\n" + " dpo sets the DPO (disable page out) in SCSI READs " + "and WRITEs\n" + " dsync sets the O_SYNC flag on open()\n" + " excl sets the O_EXCL flag on open()\n" + " ff use all 0xff bytes instead of if=IFILE (only in " + "iflags)\n" + " fua sets the FUA (force unit access) in SCSI READs " + "and WRITEs\n" + " hipri same as 'polled'; 'hipri' name is deprecated\n" + " masync set 'more async' flag on this sg device\n" + " mmap setup mmap IO on IFILE or OFILE; OFILE only " + "with noshare\n" + " mmap,mmap when used twice, doesn't call munmap()\n" + " mout_if set META_OUT_IF flag on each request\n" + " mrq_immed if mrq active, do submit non-blocking (def: " + "ordered\n" + " blocking)\n" + " mrq_svb if mrq and sg->sg copy, do shared_variable_" + "blocking\n" + " nocreat will fail rather than create OFILE\n" + " nodur turns off command duration calculations\n" + " noxfer no transfer to/from the user space\n" + " no_thresh skip checking per fd max data xfer\n" + " null does nothing, placeholder\n" + " polled set POLLED flag on command, uses blk_poll() to " + "complete\n" + " qhead queue new request at head of block queue\n" + " qtail queue new request at tail of block queue (def: " + "q at head)\n" + " random use random data instead of if=IFILE (only in " + "iflags)\n" + " same_fds each thread uses the same IFILE and OFILE(2) " + "file\n" + " descriptors (def: each threads has own file " + "descriptors)\n" + " swait this option is now ignored\n" + " v3 use v3 sg interface (def: v3 unless sg driver " + "is v4)\n" + " v4 use v4 sg interface (def: v3 unless sg driver " + "is v4)\n" + " wq_excl set SG_CTL_FLAGM_EXCL_WAITQ on this sg fd\n" + "\n" + "Copies IFILE to OFILE (and to OFILE2 if given). If IFILE and " + "OFILE are sg\ndevices 'shared' mode is selected unless " + "'noshare' is given to 'iflag=' or\n'oflag='. of2=OFILE2 uses " + "'oflag=FLAGS'. When sharing, the data stays in a\nsingle " + "in-kernel buffer which is copied (or mmap-ed) to the user " + "space\nif the 'ofreg=OFREG' is given. Use '-hhhh' for more " + "information.\n" + ); + return; +page4: + pr2serr("pack_id:\n" + "These are ascending integers, starting at 1, associated with " + "each issued\nSCSI command. When both IFILE and OFILE are sg " + "devices, then the READ in\neach read-write pair is issued an " + "even pack_id and its WRITE pair is\ngiven the pack_id one " + "higher (i.e. an odd number). This enables a\n'dmesg -w' " + "user to see that progress is being " + "made.\n\n"); + pr2serr("Debugging:\n" + "Apart from using one or more '--verbose' options which gets a " + "bit noisy\n'dmesg -w' can give a good overview " + "of what is happening.\nThat does a sg driver object tree " + "traversal that does minimal locking\nto make sure that each " + "traversal is 'safe'. So it is important to note\nthe whole " + "tree is not locked. This means for fast devices the overall\n" + "tree state may change while the traversal is occurring. For " + "example,\nit has been observed that both the read- and write- " + "sides of a request\nshare show they are in 'active' state " + "which should not be possible.\nIt occurs because the read-" + "side probably jumped out of active state and\nthe write-side " + "request entered it while some other nodes were being " + "printed.\n\n"); + pr2serr("Busy state:\n" + "Busy state (abbreviated to 'bsy' in the dmesg " + "output)\nis entered during request setup and completion. It " + "is intended to be\na temporary state. It should not block " + "but does sometimes (e.g. in\nblock_get_request()). Even so " + "that blockage should be short and if not\nthere is a " + "problem.\n\n"); + pr2serr("--verify :\n" + "For comparing IFILE with OFILE. Does repeated sequences of: " + "READ(ifile)\nand uses data returned to send to VERIFY(ofile, " + "BYTCHK=1). So the OFILE\ndevice/disk is doing the actual " + "comparison. Stops on first miscompare.\n\n"); + pr2serr("--prefetch :\n" + "Used with --verify option. Prepends a PRE-FETCH(ofile, IMMED) " + "to verify\nsequence. This should speed the trailing VERIFY by " + "making sure that\nthe data it needs for the comparison is " + "already in its cache.\n"); + return; +} + +static void +lk_print_command_len(const char *prefix, uint8_t * cmdp, int len, bool lock) +{ + if (lock) + pthread_mutex_lock(&strerr_mut); + if (prefix && *prefix) + fputs(prefix, stderr); + sg_print_command_len(cmdp, len); + if (lock) + pthread_mutex_unlock(&strerr_mut); +} + +static void +lk_chk_n_print3(const char * leadin, struct sg_io_hdr * hp, bool raw_sinfo) +{ + pthread_mutex_lock(&strerr_mut); + sg_chk_n_print3(leadin, hp, raw_sinfo); + pthread_mutex_unlock(&strerr_mut); +} + +static void +lk_chk_n_print4(const char * leadin, const struct sg_io_v4 * h4p, + bool raw_sinfo) +{ + pthread_mutex_lock(&strerr_mut); + sg_linux_sense_print(leadin, h4p->device_status, h4p->transport_status, + h4p->driver_status, (const uint8_t *)h4p->response, + h4p->response_len, raw_sinfo); + pthread_mutex_unlock(&strerr_mut); +} + +static void +hex2stderr_lk(const uint8_t * b_str, int len, int no_ascii) +{ + pthread_mutex_lock(&strerr_mut); + hex2stderr(b_str, len, no_ascii); + pthread_mutex_unlock(&strerr_mut); +} + +/* Flags decoded into abbreviations for those that are set, separated by + * '|' . */ +static char * +sg_flags_str(int flags, int b_len, char * b) +{ + int n = 0; + + if ((b_len < 1) || (! b)) + return b; + b[0] = '\0'; + if (SG_FLAG_DIRECT_IO & flags) { /* 0x1 */ + n += sg_scnpr(b + n, b_len - n, "DIO|"); + if (n >= b_len) + goto fini; + } + if (SG_FLAG_MMAP_IO & flags) { /* 0x4 */ + n += sg_scnpr(b + n, b_len - n, "MMAP|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_YIELD_TAG & flags) { /* 0x8 */ + n += sg_scnpr(b + n, b_len - n, "YTAG|"); + if (n >= b_len) + goto fini; + } + if (SG_FLAG_Q_AT_TAIL & flags) { /* 0x10 */ + n += sg_scnpr(b + n, b_len - n, "QTAI|"); + if (n >= b_len) + goto fini; + } + if (SG_FLAG_Q_AT_HEAD & flags) { /* 0x20 */ + n += sg_scnpr(b + n, b_len - n, "QHEA|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_DOUT_OFFSET & flags) { /* 0x40 */ + n += sg_scnpr(b + n, b_len - n, "DOFF|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_EVENTFD & flags) { /* 0x80 */ + n += sg_scnpr(b + n, b_len - n, "EVFD|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_COMPLETE_B4 & flags) { /* 0x100 */ + n += sg_scnpr(b + n, b_len - n, "CPL_B4|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_SIGNAL & flags) { /* 0x200 */ + n += sg_scnpr(b + n, b_len - n, "SIGNAL|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_IMMED & flags) { /* 0x400 */ + n += sg_scnpr(b + n, b_len - n, "IMM|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_POLLED & flags) { /* 0x800 */ + n += sg_scnpr(b + n, b_len - n, "POLLED|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_STOP_IF & flags) { /* 0x1000 */ + n += sg_scnpr(b + n, b_len - n, "STOPIF|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_DEV_SCOPE & flags) { /* 0x2000 */ + n += sg_scnpr(b + n, b_len - n, "DEV_SC|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_SHARE & flags) { /* 0x4000 */ + n += sg_scnpr(b + n, b_len - n, "SHARE|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_DO_ON_OTHER & flags) { /* 0x8000 */ + n += sg_scnpr(b + n, b_len - n, "DO_OTH|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_NO_DXFER & flags) { /* 0x10000 */ + n += sg_scnpr(b + n, b_len - n, "NOXFER|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_KEEP_SHARE & flags) { /* 0x20000 */ + n += sg_scnpr(b + n, b_len - n, "KEEP_SH|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_MULTIPLE_REQS & flags) { /* 0x40000 */ + n += sg_scnpr(b + n, b_len - n, "MRQS|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_ORDERED_WR & flags) { /* 0x80000 */ + n += sg_scnpr(b + n, b_len - n, "OWR|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_REC_ORDER & flags) { /* 0x100000 */ + n += sg_scnpr(b + n, b_len - n, "REC_O|"); + if (n >= b_len) + goto fini; + } + if (SGV4_FLAG_META_OUT_IF & flags) { /* 0x200000 */ + n += sg_scnpr(b + n, b_len - n, "MOUT_IF|"); + if (n >= b_len) + goto fini; + } + if (0 == n) + n += sg_scnpr(b + n, b_len - n, "<none>"); +fini: + if (n < b_len) { /* trim trailing '\' */ + if ('|' == b[n - 1]) + b[n - 1] = '\0'; + } else if ('|' == b[b_len - 1]) + b[b_len - 1] = '\0'; + return b; +} + +/* Info field decoded into abbreviations for those bits that are set, + * separated by '|' . */ +static char * +sg_info_str(int info, int b_len, char * b) +{ + int n = 0; + + if ((b_len < 1) || (! b)) + return b; + b[0] = '\0'; + if (SG_INFO_CHECK & info) { /* 0x1 */ + n += sg_scnpr(b + n, b_len - n, "CHK|"); + if (n >= b_len) + goto fini; + } + if (SG_INFO_DIRECT_IO & info) { /* 0x2 */ + n += sg_scnpr(b + n, b_len - n, "DIO|"); + if (n >= b_len) + goto fini; + } + if (SG_INFO_MIXED_IO & info) { /* 0x4 */ + n += sg_scnpr(b + n, b_len - n, "MIO|"); + if (n >= b_len) + goto fini; + } + if (SG_INFO_DEVICE_DETACHING & info) { /* 0x8 */ + n += sg_scnpr(b + n, b_len - n, "DETA|"); + if (n >= b_len) + goto fini; + } + if (SG_INFO_ABORTED & info) { /* 0x10 */ + n += sg_scnpr(b + n, b_len - n, "ABRT|"); + if (n >= b_len) + goto fini; + } + if (SG_INFO_MRQ_FINI & info) { /* 0x20 */ + n += sg_scnpr(b + n, b_len - n, "MRQF|"); + if (n >= b_len) + goto fini; + } +fini: + if (n < b_len) { /* trim trailing '\' */ + if ('|' == b[n - 1]) + b[n - 1] = '\0'; + } else if ('|' == b[b_len - 1]) + b[b_len - 1] = '\0'; + return b; +} + +static void +v4hdr_out_lk(const char * leadin, const sg_io_v4 * h4p, int id) +{ + char b[80]; + + pthread_mutex_lock(&strerr_mut); + if (leadin) + pr2serr("%s [id=%d]:\n", leadin, id); + if (('Q' != h4p->guard) || (0 != h4p->protocol) || + (0 != h4p->subprotocol)) + pr2serr(" <<<sg_io_v4 _NOT_ properly set>>>\n"); + pr2serr(" pointers: cdb=%s sense=%s din=%p dout=%p\n", + (h4p->request ? "y" : "NULL"), (h4p->response ? "y" : "NULL"), + (void *)h4p->din_xferp, (void *)h4p->dout_xferp); + pr2serr(" lengths: cdb=%u sense=%u din=%u dout=%u\n", + h4p->request_len, h4p->max_response_len, h4p->din_xfer_len, + h4p->dout_xfer_len); + pr2serr(" flags=0x%x request_extra{pack_id}=%d\n", + h4p->flags, h4p->request_extra); + pr2serr(" flags set: %s\n", sg_flags_str(h4p->flags, sizeof(b), b)); + pr2serr(" %s OUT fields:\n", leadin); + pr2serr(" response_len=%d driver/transport/device_status=" + "0x%x/0x%x/0x%x\n", h4p->response_len, h4p->driver_status, + h4p->transport_status, h4p->device_status); + pr2serr(" info=0x%x din_resid=%u dout_resid=%u spare_out=%u " + "dur=%u\n", + h4p->info, h4p->din_resid, h4p->dout_resid, h4p->spare_out, + h4p->duration); + pthread_mutex_unlock(&strerr_mut); +} + +static void +fetch_sg_version(void) +{ + FILE * fp; + char b[96]; + + have_sg_version = false; + sg_version = 0; + fp = fopen(PROC_SCSI_SG_VERSION, "r"); + if (fp && fgets(b, sizeof(b) - 1, fp)) { + if (1 == sscanf(b, "%d", &sg_version)) + have_sg_version = !!sg_version; + } else { + int j, k, l; + + if (fp) + fclose(fp); + fp = fopen(SYS_SCSI_SG_VERSION, "r"); + if (fp && fgets(b, sizeof(b) - 1, fp)) { + if (3 == sscanf(b, "%d.%d.%d", &j, &k, &l)) { + sg_version = (j * 10000) + (k * 100) + l; + have_sg_version = !!sg_version; + } + } + } + if (fp) + fclose(fp); +} + +static void +calc_duration_throughput(int contin) +{ + struct timeval end_tm, res_tm; + double a, b; + + gettimeofday(&end_tm, NULL); + res_tm.tv_sec = end_tm.tv_sec - start_tm.tv_sec; + res_tm.tv_usec = end_tm.tv_usec - start_tm.tv_usec; + if (res_tm.tv_usec < 0) { + --res_tm.tv_sec; + res_tm.tv_usec += 1000000; + } + a = res_tm.tv_sec; + a += (0.000001 * res_tm.tv_usec); + b = (double)gcoll.bs * (dd_count - gcoll.out_rem_count.load()); + pr2serr("time to %s data %s %d.%06d secs", + (gcoll.verify ? "verify" : "copy"), (contin ? "so far" : "was"), + (int)res_tm.tv_sec, (int)res_tm.tv_usec); + if ((a > 0.00001) && (b > 511)) + pr2serr(", %.2f MB/sec\n", b / (a * 1000000.0)); + else + pr2serr("\n"); +} + +static void +print_stats(const char * str) +{ + int64_t infull; + + if (0 != gcoll.out_rem_count.load()) + pr2serr(" remaining block count=%" PRId64 "\n", + gcoll.out_rem_count.load()); + infull = dd_count - gcoll.in_rem_count.load(); + pr2serr("%s%" PRId64 "+%d records in\n", str, + infull - gcoll.in_partial.load(), gcoll.in_partial.load()); + + if (gcoll.out_type == FT_DEV_NULL) + pr2serr("%s0+0 records out\n", str); + else { + int64_t outfull = dd_count - gcoll.out_rem_count.load(); + + pr2serr("%s%" PRId64 "+%d records %s\n", str, + outfull - gcoll.out_partial.load(), gcoll.out_partial.load(), + (gcoll.verify ? "verified" : "out")); + } +} + +static void +interrupt_handler(int sig) +{ + struct sigaction sigact; + + sigact.sa_handler = SIG_DFL; + sigemptyset(&sigact.sa_mask); + sigact.sa_flags = 0; + sigaction(sig, &sigact, NULL); + pr2serr("Interrupted by signal,"); + if (do_time > 0) + calc_duration_throughput(0); + print_stats(""); + kill(getpid (), sig); +} + +static void +siginfo_handler(int sig) +{ + if (sig) { ; } /* unused, dummy to suppress warning */ + pr2serr("Progress report, continuing ...\n"); + if (do_time > 0) + calc_duration_throughput(1); + print_stats(" "); +} + +static void +siginfo2_handler(int sig) +{ + struct global_collection * clp = &gcoll; + + if (sig) { ; } /* unused, dummy to suppress warning */ + pr2serr("Progress report, continuing ...\n"); + if (do_time > 0) + calc_duration_throughput(1); + print_stats(" "); + pr2serr("Send broadcast on out_sync_cv condition variable\n"); + pthread_cond_broadcast(&clp->out_sync_cv); +} + +static void +install_handler(int sig_num, void (*sig_handler) (int sig)) +{ + struct sigaction sigact; + sigaction (sig_num, NULL, &sigact); + if (sigact.sa_handler != SIG_IGN) + { + sigact.sa_handler = sig_handler; + sigemptyset (&sigact.sa_mask); + sigact.sa_flags = 0; + sigaction (sig_num, &sigact, NULL); + } +} + +#ifdef SG_LIB_ANDROID +static void +thread_exit_handler(int sig) +{ + pthread_exit(0); +} +#endif + +/* Make safe_strerror() thread safe */ +static char * +tsafe_strerror(int code, char * ebp) +{ + char * cp; + + pthread_mutex_lock(&strerr_mut); + cp = safe_strerror(code); + strncpy(ebp, cp, STRERR_BUFF_LEN); + pthread_mutex_unlock(&strerr_mut); + + ebp[strlen(ebp)] = '\0'; + return ebp; +} + + +/* Following macro from D.R. Butenhof's POSIX threads book: + * ISBN 0-201-63392-2 . [Highly recommended book.] Changed __FILE__ + * to __func__ */ +#define err_exit(code,text) do { \ + char strerr_buff[STRERR_BUFF_LEN + 1]; \ + pr2serr("%s at \"%s\":%d: %s\n", \ + text, __func__, __LINE__, tsafe_strerror(code, strerr_buff)); \ + exit(1); \ + } while (0) + + +static int +dd_filetype(const char * filename, off_t & st_size) +{ + struct stat st; + size_t len = strlen(filename); + + if ((1 == len) && ('.' == filename[0])) + return FT_DEV_NULL; + if (stat(filename, &st) < 0) + return FT_ERROR; + if (S_ISCHR(st.st_mode)) { + if ((MEM_MAJOR == major(st.st_rdev)) && + ((DEV_NULL_MINOR_NUM == minor(st.st_rdev)) || + (DEV_ZERO_MINOR_NUM == minor(st.st_rdev)))) + return FT_DEV_NULL; /* treat /dev/null + /dev/zero the same */ + if (SCSI_GENERIC_MAJOR == major(st.st_rdev)) + return FT_SG; + if (SCSI_TAPE_MAJOR == major(st.st_rdev)) + return FT_ST; + return FT_CHAR; + } else if (S_ISBLK(st.st_mode)) + return FT_BLOCK; + else if (S_ISFIFO(st.st_mode)) + return FT_FIFO; + st_size = st.st_size; + return FT_OTHER; +} + +static inline void +stop_both(struct global_collection * clp) +{ + clp->in_stop = true; + clp->out_stop = true; +} + +/* Return of 0 -> success, see sg_ll_read_capacity*() otherwise */ +static int +scsi_read_capacity(int sg_fd, int64_t * num_sect, int * sect_sz) +{ + int res; + uint8_t rcBuff[RCAP16_REPLY_LEN] = {}; + + res = sg_ll_readcap_10(sg_fd, 0, 0, rcBuff, READ_CAP_REPLY_LEN, false, 0); + if (0 != res) + return res; + + if ((0xff == rcBuff[0]) && (0xff == rcBuff[1]) && (0xff == rcBuff[2]) && + (0xff == rcBuff[3])) { + + res = sg_ll_readcap_16(sg_fd, 0, 0, rcBuff, RCAP16_REPLY_LEN, false, + 0); + if (0 != res) + return res; + *num_sect = sg_get_unaligned_be64(rcBuff + 0) + 1; + *sect_sz = sg_get_unaligned_be32(rcBuff + 8); + } else { + /* take care not to sign extend values > 0x7fffffff */ + *num_sect = (int64_t)sg_get_unaligned_be32(rcBuff + 0) + 1; + *sect_sz = sg_get_unaligned_be32(rcBuff + 4); + } + return 0; +} + +/* Return of 0 -> success, -1 -> failure. BLKGETSIZE64, BLKGETSIZE and */ +/* BLKSSZGET macros problematic (from <linux/fs.h> or <sys/mount.h>). */ +static int +read_blkdev_capacity(int sg_fd, int64_t * num_sect, int * sect_sz) +{ +#ifdef BLKSSZGET + if ((ioctl(sg_fd, BLKSSZGET, sect_sz) < 0) && (*sect_sz > 0)) { + perror("BLKSSZGET ioctl error"); + return -1; + } else { + #ifdef BLKGETSIZE64 + uint64_t ull; + + if (ioctl(sg_fd, BLKGETSIZE64, &ull) < 0) { + + perror("BLKGETSIZE64 ioctl error"); + return -1; + } + *num_sect = ((int64_t)ull / (int64_t)*sect_sz); + #else + unsigned long ul; + + if (ioctl(sg_fd, BLKGETSIZE, &ul) < 0) { + perror("BLKGETSIZE ioctl error"); + return -1; + } + *num_sect = (int64_t)ul; + #endif + } + return 0; +#else + *num_sect = 0; + *sect_sz = 0; + return -1; +#endif +} + +static int +system_wrapper(const char * cmd) +{ + int res; + + res = system(cmd); + if (WIFSIGNALED(res) && + (WTERMSIG(res) == SIGINT || WTERMSIG(res) == SIGQUIT)) + raise(WTERMSIG(res)); + return WEXITSTATUS(res); +} + +/* Has an infinite loop doing a timed wait for any signals in sig_set. After + * each timeout (300 ms) checks if the most_recent_pack_id atomic integer + * has changed. If not after another two timeouts announces a stall has + * been detected. If shutting down atomic is true breaks out of loop and + * shuts down this thread. Other than that, this thread is normally cancelled + * by the main thread, after other threads have exited. */ +static void * +sig_listen_thread(void * v_clp) +{ + bool stall_reported = false; + int prev_pack_id = 0; + struct timespec ts; + struct timespec * tsp = &ts; + struct global_collection * clp = (struct global_collection *)v_clp; + uint32_t ict_ms = (clp->sdt_ict ? clp->sdt_ict : DEF_SDT_ICT_MS); + + tsp->tv_sec = ict_ms / 1000; + tsp->tv_nsec = (ict_ms % 1000) * 1000 * 1000; /* DEF_SDT_ICT_MS */ + while (1) { + int sig_number = sigtimedwait(&signal_set, NULL, tsp); + + if (sig_number < 0) { + int err = errno; + + /* EAGAIN implies a timeout */ + if ((EAGAIN == err) && (clp->sdt_crt > 0)) { + int pack_id = mono_pack_id.load(); + + if ((pack_id > 0) && (pack_id == prev_pack_id)) { + if (! stall_reported) { + stall_reported = true; + tsp->tv_sec = clp->sdt_crt; + tsp->tv_nsec = 0; + pr2serr_lk("%s: first stall at pack_id=%d detected\n", + __func__, pack_id); + } else + pr2serr_lk("%s: subsequent stall at pack_id=%d\n", + __func__, pack_id); + // following command assumes linux bash or similar shell + system_wrapper("cat /proc/scsi/sg/debug >> /dev/stderr\n"); + // system_wrapper("/usr/bin/dmesg\n"); + } else + prev_pack_id = pack_id; + } else if (EAGAIN != err) + pr2serr_lk("%s: sigtimedwait() errno=%d\n", __func__, err); + } + if (SIGINT == sig_number) { + pr2serr_lk("%sinterrupted by SIGINT\n", my_name); + stop_both(clp); + pthread_cond_broadcast(&clp->out_sync_cv); + sigprocmask(SIG_SETMASK, &orig_signal_set, NULL); + raise(SIGINT); + break; + } + if (SIGUSR2 == sig_number) { + if (clp->verbose > 2) + pr2serr_lk("%s: interrupted by SIGUSR2\n", __func__); + break; + } + if (shutting_down) + break; + } /* end of while loop */ + if (clp->verbose > 3) + pr2serr_lk("%s: exiting\n", __func__); + + return NULL; +} + +static void * +mrq_abort_thread(void * v_maip) +{ + int res, err; + int n = 0; + int seed; + unsigned int rn; + Mrq_abort_info l_mai = *(Mrq_abort_info *)v_maip; + struct sg_io_v4 ctl_v4 {}; + +#ifdef HAVE_GETRANDOM + { + ssize_t ssz = getrandom(&seed, sizeof(seed), GRND_NONBLOCK); + + if (ssz < (ssize_t)sizeof(seed)) { + pr2serr("getrandom() failed, ret=%d\n", (int)ssz); + seed = (int)time(NULL); + } + } +#else + seed = (int)time(NULL); /* use seconds since epoch as proxy */ +#endif + if (l_mai.debug) + pr2serr_lk("%s: from_id=%d: to abort mrq_pack_id=%d\n", __func__, + l_mai.from_tid, l_mai.mrq_id); + res = ioctl(l_mai.fd, SG_GET_NUM_WAITING, &n); + ++num_waiting_calls; + if (res < 0) { + err = errno; + pr2serr_lk("%s: ioctl(SG_GET_NUM_WAITING) failed: %s [%d]\n", + __func__, safe_strerror(err), err); + } else if (l_mai.debug) + pr2serr_lk("%s: num_waiting=%d\n", __func__, n); + + Rand_uint * ruip = new Rand_uint(5, 500, seed); + struct timespec tspec = {0, 4000 /* 4 usecs */}; + rn = ruip->get(); + tspec.tv_nsec = rn * 1000; + if (l_mai.debug > 1) + pr2serr_lk("%s: /dev/urandom seed=0x%x delay=%u microsecs\n", + __func__, seed, rn); + if (rn >= 20) + nanosleep(&tspec, NULL); + else if (l_mai.debug > 1) + pr2serr_lk("%s: skipping nanosleep cause delay < 20 usecs\n", + __func__); + + ctl_v4.guard = 'Q'; + ctl_v4.flags = SGV4_FLAG_MULTIPLE_REQS; + ctl_v4.request_extra = l_mai.mrq_id; + ++num_mrq_abort_req; + res = ioctl(l_mai.fd, SG_IOABORT, &ctl_v4); + if (res < 0) { + err = errno; + if (ENODATA == err) + pr2serr_lk("%s: ioctl(SG_IOABORT) no match on " + "MRQ pack_id=%d\n", __func__, l_mai.mrq_id); + else + pr2serr_lk("%s: MRQ ioctl(SG_IOABORT) failed: %s [%d]\n", + __func__, safe_strerror(err), err); + } else { + ++num_mrq_abort_req_success; + if (l_mai.debug > 1) + pr2serr_lk("%s: from_id=%d sent ioctl(SG_IOABORT) on MRQ rq_id=" + "%d, success\n", __func__, l_mai.from_tid, + l_mai.mrq_id); + } + delete ruip; + return NULL; +} + +static bool +sg_share_prepare(int write_side_fd, int read_side_fd, int id, bool vb_b) +{ + struct sg_extended_info sei {}; + struct sg_extended_info * seip = &sei; + + seip->sei_wr_mask |= SG_SEIM_SHARE_FD; + seip->sei_rd_mask |= SG_SEIM_SHARE_FD; + seip->share_fd = read_side_fd; + if (ioctl(write_side_fd, SG_SET_GET_EXTENDED, seip) < 0) { + pr2serr_lk("tid=%d: ioctl(EXTENDED(shared_fd=%d), failed " + "errno=%d %s\n", id, read_side_fd, errno, + strerror(errno)); + return false; + } + if (vb_b) + pr2serr_lk("%s: tid=%d: ioctl(EXTENDED(shared_fd)) ok, " + "read_side_fd=%d, write_side_fd=%d\n", __func__, + id, read_side_fd, write_side_fd); + return true; +} + +static void +sg_unshare(int sg_fd, int id, bool vb_b) +{ + struct sg_extended_info sei {}; + struct sg_extended_info * seip = &sei; + + seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS; + seip->sei_rd_mask |= SG_SEIM_CTL_FLAGS; + seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_UNSHARE; + seip->ctl_flags |= SG_CTL_FLAGM_UNSHARE; /* needs to be set to unshare */ + if (ioctl(sg_fd, SG_SET_GET_EXTENDED, seip) < 0) { + pr2serr_lk("tid=%d: ioctl(EXTENDED(UNSHARE), failed errno=%d %s\n", + id, errno, strerror(errno)); + return; + } + if (vb_b) + pr2serr_lk("tid=%d: ioctl(UNSHARE) ok\n", id); +} + +static void +sg_noshare_enlarge(int sg_fd, bool vb_b) +{ + if (sg_version_ge_40045) { + struct sg_extended_info sei {}; + struct sg_extended_info * seip = &sei; + + sei.sei_wr_mask |= SG_SEIM_TOT_FD_THRESH; + seip->tot_fd_thresh = 96 * 1024 * 1024; + if (ioctl(sg_fd, SG_SET_GET_EXTENDED, seip) < 0) { + pr2serr_lk("%s: ioctl(EXTENDED(TOT_FD_THRESH), failed errno=%d " + "%s\n", __func__, errno, strerror(errno)); + return; + } + if (vb_b) + pr2serr_lk("ioctl(TOT_FD_THRESH) ok\n"); + } +} + +static void +sg_take_snap(int sg_fd, int id, bool vb_b) +{ + struct sg_extended_info sei {}; + struct sg_extended_info * seip = &sei; + + seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS; + seip->sei_rd_mask |= SG_SEIM_CTL_FLAGS; + seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_SNAP_DEV; + seip->ctl_flags &= ~SG_CTL_FLAGM_SNAP_DEV; /* 0 --> append */ + if (ioctl(sg_fd, SG_SET_GET_EXTENDED, seip) < 0) { + pr2serr_lk("tid=%d: ioctl(EXTENDED(SNAP_DEV), failed errno=%d %s\n", + id, errno, strerror(errno)); + return; + } + if (vb_b) + pr2serr_lk("tid=%d: ioctl(SNAP_DEV) ok\n", id); +} + +static void +cleanup_in(void * v_clp) +{ + struct global_collection * clp = (struct global_collection *)v_clp; + + pr2serr("thread cancelled while in mutex held\n"); + stop_both(clp); + pthread_mutex_unlock(&clp->in_mutex); + pthread_cond_broadcast(&clp->out_sync_cv); +} + +static void +cleanup_out(void * v_clp) +{ + struct global_collection * clp = (struct global_collection *)v_clp; + + pr2serr("thread cancelled while out_mutex held\n"); + stop_both(clp); + pthread_mutex_unlock(&clp->out_mutex); + pthread_cond_broadcast(&clp->out_sync_cv); +} + +static void inline buffp_onto_next(Rq_elem * rep) +{ + struct global_collection * clp = rep->clp; + + if ((clp->nmrqs > 0) && clp->unbalanced_mrq) { + ++rep->mrq_index; + if (rep->mrq_index >= clp->nmrqs) + rep->mrq_index = 0; /* wrap */ + } +} + +static inline uint8_t * +get_buffp(Rq_elem * rep) +{ + struct global_collection * clp = rep->clp; + + if ((clp->nmrqs > 0) && clp->unbalanced_mrq && (rep->mrq_index > 0)) + return rep->buffp + (rep->mrq_index * clp->bs * clp->bpt); + else + return rep->buffp; +} + +static void * +read_write_thread(void * v_tip) +{ + Thread_info * tip; + struct global_collection * clp; + Rq_elem rel {}; + Rq_elem * rep = &rel; + int n, sz, blocks, status, vb, err, res, wr_blks, c_addr; + int num_sg = 0; + int64_t my_index; + volatile bool stop_after_write = false; + bool own_infd = false; + bool in_is_sg, in_mmap, out_is_sg, out_mmap; + bool own_outfd = false; + bool own_out2fd = false; + bool share_and_ofreg; + mrq_arr_t deferred_arr; /* MRQ deferred array (vector) */ + + tip = (Thread_info *)v_tip; + clp = tip->gcp; + vb = clp->verbose; + rep->bs = clp->bs; + sz = clp->bpt * rep->bs; + c_addr = clp->chkaddr; + in_is_sg = (FT_SG == clp->in_type); + in_mmap = (in_is_sg && (clp->in_flags.mmap > 0)); + out_is_sg = (FT_SG == clp->out_type); + out_mmap = (out_is_sg && (clp->out_flags.mmap > 0)); + /* Following clp members are constant during lifetime of thread */ + rep->clp = clp; + rep->id = tip->id; + if (vb > 2) + pr2serr_lk("%d <-- Starting worker thread\n", rep->id); + if (! (in_mmap || out_mmap)) { + n = sz; + if (clp->unbalanced_mrq) + n *= clp->nmrqs; + rep->buffp = sg_memalign(n, 0 /* page align */, &rep->alloc_bp, + false); + if (NULL == rep->buffp) + err_exit(ENOMEM, "out of memory creating user buffers\n"); + } + rep->infd = clp->infd; + rep->outfd = clp->outfd; + rep->out2fd = clp->out2fd; + rep->outregfd = clp->outregfd; + rep->rep_count = 0; + if (clp->unbalanced_mrq && (clp->nmrqs > 0)) + rep->mrq_index = clp->nmrqs - 1; + + if (rep->infd == rep->outfd) { + if (in_is_sg) + rep->same_sg = true; + } else if (in_is_sg && out_is_sg) + rep->both_sg = true; + else if (in_is_sg) + rep->only_in_sg = true; + else if (out_is_sg) + rep->only_out_sg = true; + + if (clp->in_flags.random) { +#ifdef HAVE_GETRANDOM + ssize_t ssz = getrandom(&rep->seed, sizeof(rep->seed), GRND_NONBLOCK); + + if (ssz < (ssize_t)sizeof(rep->seed)) { + pr2serr_lk("thread=%d: getrandom() failed, ret=%d\n", + rep->id, (int)ssz); + rep->seed = (long)time(NULL); + } +#else + rep->seed = (long)time(NULL); /* use seconds since epoch as proxy */ +#endif + if (vb > 1) + pr2serr_lk("thread=%d: seed=%ld\n", rep->id, rep->seed); +#ifdef HAVE_SRAND48_R + srand48_r(rep->seed, &rep->drand); +#else + srand48(rep->seed); +#endif + } + if (clp->in_flags.same_fds || clp->out_flags.same_fds) + ; + else { + int fd; + + if (in_is_sg && clp->infp) { + fd = sg_in_open(clp, clp->infp, (in_mmap ? &rep->buffp : NULL), + (in_mmap ? &rep->mmap_len : NULL)); + if (fd < 0) + goto fini; + rep->infd = fd; + rep->mmap_active = in_mmap ? clp->in_flags.mmap : 0; + if (in_mmap && (vb > 4)) + pr2serr_lk("thread=%d: mmap buffp=%p\n", rep->id, rep->buffp); + own_infd = true; + ++num_sg; + if (vb > 2) + pr2serr_lk("thread=%d: opened local sg IFILE\n", rep->id); + } + if (out_is_sg && clp->outfp) { + fd = sg_out_open(clp, clp->outfp, (out_mmap ? &rep->buffp : NULL), + (out_mmap ? &rep->mmap_len : NULL)); + if (fd < 0) + goto fini; + rep->outfd = fd; + if (! rep->mmap_active) + rep->mmap_active = out_mmap ? clp->out_flags.mmap : 0; + if (out_mmap && (vb > 4)) + pr2serr_lk("thread=%d: mmap buffp=%p\n", rep->id, rep->buffp); + own_outfd = true; + ++num_sg; + if (vb > 2) + pr2serr_lk("thread=%d: opened local sg OFILE\n", rep->id); + } + if ((FT_SG == clp->out2_type) && clp->out2fp) { + fd = sg_out_open(clp, clp->out2fp, + (out_mmap ? &rep->buffp : NULL), + (out_mmap ? &rep->mmap_len : NULL)); + if (fd < 0) + goto fini; + rep->out2fd = fd; + own_out2fd = true; + if (vb > 2) + pr2serr_lk("thread=%d: opened local sg OFILE2\n", rep->id); + } + } + if (vb > 2) { + if (in_is_sg && (! own_infd)) + pr2serr_lk("thread=%d: using global sg IFILE, fd=%d\n", rep->id, + rep->infd); + if (out_is_sg && (! own_outfd)) + pr2serr_lk("thread=%d: using global sg OFILE, fd=%d\n", rep->id, + rep->outfd); + if ((FT_SG == clp->out2_type) && (! own_out2fd)) + pr2serr_lk("thread=%d: using global sg OFILE2, fd=%d\n", rep->id, + rep->out2fd); + } + if (!sg_version_ge_40045) { + if (vb > 4) + pr2serr_lk("thread=%d: Skipping share because driver too old\n", + rep->id); + } else if (clp->noshare) { + if (vb > 4) + pr2serr_lk("thread=%d: Skipping IFILE share with OFILE due to " + "noshare=1\n", rep->id); + } else if (sg_version_ge_40045 && in_is_sg && out_is_sg) + rep->has_share = sg_share_prepare(rep->outfd, rep->infd, rep->id, + vb > 9); + if (vb > 9) + pr2serr_lk("tid=%d, has_share=%s\n", rep->id, + (rep->has_share ? "true" : "false")); + share_and_ofreg = (rep->has_share && (rep->outregfd >= 0)); + + /* vvvvvvvvvvvvvv Main segment copy loop vvvvvvvvvvvvvvvvvvvvvvv */ + while (1) { + rep->wr = false; + my_index = atomic_fetch_add(&pos_index, (long int)clp->bpt); + /* Start of READ half of a segment */ + buffp_onto_next(rep); + status = pthread_mutex_lock(&clp->in_mutex); + if (0 != status) err_exit(status, "lock in_mutex"); + + if (dd_count >= 0) { + if (my_index >= dd_count) { + status = pthread_mutex_unlock(&clp->in_mutex); + if (0 != status) err_exit(status, "unlock in_mutex"); + if ((clp->nmrqs > 0) && (deferred_arr.first.size() > 0)) { + if (vb > 2) + pr2serr_lk("thread=%d: tail-end my_index>=dd_count, " + "to_do=%u\n", rep->id, + (uint32_t)deferred_arr.first.size()); + res = sgh_do_deferred_mrq(rep, deferred_arr); + if (res) + pr2serr_lk("%s tid=%d: sgh_do_deferred_mrq failed\n", + __func__, rep->id); + } + break; /* at or beyond end, so leave loop >>>>>>>>>> */ + } else if ((my_index + clp->bpt) > dd_count) + blocks = dd_count - my_index; + else + blocks = clp->bpt; + } else + blocks = clp->bpt; + + rep->iblk = clp->skip + my_index; + rep->oblk = clp->seek + my_index; + rep->num_blks = blocks; + + // clp->in_blk += blocks; + // clp->in_count -= blocks; + + pthread_cleanup_push(cleanup_in, (void *)clp); + if (in_is_sg) + sg_in_rd_cmd(clp, rep, deferred_arr); + else { + stop_after_write = normal_in_rd(rep, blocks); + status = pthread_mutex_unlock(&clp->in_mutex); + if (0 != status) err_exit(status, "unlock in_mutex"); + } + if (c_addr && (rep->bs > 3)) { + int k, j, off, num; + uint32_t addr = (uint32_t)rep->iblk; + + num = (1 == c_addr) ? 4 : (rep->bs - 3); + for (k = 0, off = 0; k < blocks; ++k, ++addr, off += rep->bs) { + for (j = 0; j < num; j += 4) { + if (addr != sg_get_unaligned_be32(rep->buffp + off + j)) + break; + } + if (j < num) + break; + } + if (k < blocks) { + pr2serr("%s: chkaddr failure at addr=0x%x\n", __func__, addr); + exit_status = SG_LIB_CAT_MISCOMPARE; + ++num_miscompare; + stop_both(clp); + } + } + pthread_cleanup_pop(0); + ++rep->rep_count; + + /* Start of WRITE part of a segment */ + rep->wr = true; + status = pthread_mutex_lock(&clp->out_mutex); + if (0 != status) err_exit(status, "lock out_mutex"); + + /* Make sure the OFILE (+ OFREG) are in same sequence as IFILE */ + if (clp->in_flags.random) + goto skip_force_out_sequence; + if ((rep->outregfd < 0) && in_is_sg && out_is_sg) + goto skip_force_out_sequence; + if (share_and_ofreg || (FT_DEV_NULL != clp->out_type)) { + while ((! clp->out_stop.load()) && + (rep->oblk != clp->out_blk.load())) { + /* if write would be out of sequence then wait */ + pthread_cleanup_push(cleanup_out, (void *)clp); + status = pthread_cond_wait(&clp->out_sync_cv, &clp->out_mutex); + if (0 != status) err_exit(status, "cond out_sync_cv"); + pthread_cleanup_pop(0); + } + } + +skip_force_out_sequence: + if (clp->out_stop.load() || (clp->out_count.load() <= 0)) { + if (! clp->out_stop.load()) + clp->out_stop = true; + status = pthread_mutex_unlock(&clp->out_mutex); + if (0 != status) err_exit(status, "unlock out_mutex"); + break; /* stop requested so leave loop >>>>>>>>>> */ + } + if (stop_after_write) + clp->out_stop = true; + + clp->out_count -= blocks; + clp->out_blk += blocks; + + pthread_cleanup_push(cleanup_out, (void *)clp); + if (rep->outregfd >= 0) { + res = write(rep->outregfd, get_buffp(rep), + rep->bs * rep->num_blks); + err = errno; + if (res < 0) + pr2serr_lk("%s: tid=%d: write(outregfd) failed: %s\n", + __func__, rep->id, strerror(err)); + else if (vb > 9) + pr2serr_lk("%s: tid=%d: write(outregfd), fd=%d, num_blks=%d" + "\n", __func__, rep->id, rep->outregfd, + rep->num_blks); + } + /* Output to OFILE */ + wr_blks = rep->num_blks; + if (out_is_sg) { + sg_out_wr_cmd(rep, deferred_arr, false, clp->prefetch); + ++rep->rep_count; + } else if (FT_DEV_NULL == clp->out_type) { + /* skip actual write operation */ + wr_blks = 0; + clp->out_rem_count -= blocks; + status = pthread_mutex_unlock(&clp->out_mutex); + if (0 != status) err_exit(status, "unlock out_mutex"); + } else { + normal_out_wr(rep, blocks); + status = pthread_mutex_unlock(&clp->out_mutex); + if (0 != status) err_exit(status, "unlock out_mutex"); + ++rep->rep_count; + } + pthread_cleanup_pop(0); + + /* Output to OFILE2 if sg device */ + if ((clp->out2fd >= 0) && (FT_SG == clp->out2_type)) { + pthread_cleanup_push(cleanup_out, (void *)clp); + status = pthread_mutex_lock(&clp->out2_mutex); + if (0 != status) err_exit(status, "lock out2_mutex"); + /* releases out2_mutex mid operation */ + sg_out_wr_cmd(rep, deferred_arr, true, false); + + pthread_cleanup_pop(0); + } + if (0 == rep->num_blks) { + if ((clp->nmrqs > 0) && (deferred_arr.first.size() > 0)) { + if (wr_blks > 0) + rep->out_mrq_q_blks += wr_blks; + if (vb > 2) + pr2serr_lk("thread=%d: tail-end, to_do=%u\n", rep->id, + (uint32_t)deferred_arr.first.size()); + res = sgh_do_deferred_mrq(rep, deferred_arr); + if (res) + pr2serr_lk("%s tid=%d: sgh_do_deferred_mrq failed\n", + __func__, rep->id); + } + clp->out_stop = true; + stop_after_write = true; + break; /* read nothing so leave loop >>>>>>>>>> */ + } + // if ((! rep->has_share) && (FT_DEV_NULL != clp->out_type)) + pthread_cond_broadcast(&clp->out_sync_cv); + if (stop_after_write) + break; /* leaving main loop >>>>>>>>> */ + } /* ^^^^^^^^^^ end of main while loop which copies segments ^^^^^^ */ + + status = pthread_mutex_lock(&clp->in_mutex); + if (0 != status) err_exit(status, "lock in_mutex"); + if (! clp->in_stop.load()) + clp->in_stop = true; /* flag other workers to stop */ + status = pthread_mutex_unlock(&clp->in_mutex); + if (0 != status) err_exit(status, "unlock in_mutex"); + +fini: + if ((1 == rep->mmap_active) && (rep->mmap_len > 0)) { + if (munmap(rep->buffp, rep->mmap_len) < 0) { + err = errno; + char bb[STRERR_BUFF_LEN + 1]; + + pr2serr_lk("thread=%d: munmap() failed: %s\n", rep->id, + tsafe_strerror(err, bb)); + } + if (vb > 4) + pr2serr_lk("thread=%d: munmap(%p, %d)\n", rep->id, rep->buffp, + rep->mmap_len); + rep->mmap_active = 0; + } + if (rep->alloc_bp) { + free(rep->alloc_bp); + rep->alloc_bp = NULL; + rep->buffp = NULL; + } + + if (sg_version_ge_40045) { + if (clp->noshare) { + if ((clp->nmrqs > 0) && clp->unshare) + sg_unshare(rep->infd, rep->id, vb > 9); + } else if (in_is_sg && out_is_sg) + if (clp->unshare) + sg_unshare(rep->infd, rep->id, vb > 9); + } + if (own_infd && (rep->infd >= 0)) { + if (vb && in_is_sg) { + ++num_waiting_calls; + if (ioctl(rep->infd, SG_GET_NUM_WAITING, &n) >= 0) { + if (n > 0) + pr2serr_lk("%s: tid=%d: num_waiting=%d prior close(in)\n", + __func__, rep->id, n); + } else { + err = errno; + pr2serr_lk("%s: [%d] ioctl(SG_GET_NUM_WAITING) errno=%d: " + "%s\n", __func__, rep->id, err, strerror(err)); + } + } + close(rep->infd); + } + if (own_outfd && (rep->outfd >= 0)) { + if (vb && out_is_sg) { + ++num_waiting_calls; + if (ioctl(rep->outfd, SG_GET_NUM_WAITING, &n) >= 0) { + if (n > 0) + pr2serr_lk("%s: tid=%d: num_waiting=%d prior " + "close(out)\n", __func__, rep->id, n); + } else { + err = errno; + pr2serr_lk("%s: [%d] ioctl(SG_GET_NUM_WAITING) errno=%d: " + "%s\n", __func__, rep->id, err, strerror(err)); + } + } + close(rep->outfd); + } + if (own_out2fd && (rep->out2fd >= 0)) + close(rep->out2fd); + pthread_cond_broadcast(&clp->out_sync_cv); + return stop_after_write ? NULL : clp; +} + +/* N.B. A return of true means it wants to stop the copy. So false is the + * 'good' reply (i.e. keep going). */ +static bool +normal_in_rd(Rq_elem * rep, int blocks) +{ + struct global_collection * clp = rep->clp; + bool stop_after_write = false; + bool same_fds = clp->in_flags.same_fds || clp->out_flags.same_fds; + int res; + + if (clp->verbose > 4) + pr2serr_lk("%s: tid=%d: iblk=%" PRIu64 ", blocks=%d\n", __func__, + rep->id, rep->iblk, blocks); + if (FT_RANDOM_0_FF == clp->in_type) { + int k, j; + const int jbump = sizeof(uint32_t); + long rn; + uint8_t * bp; + + if (clp->in_flags.zero && clp->in_flags.ff && (rep->bs >= 4)) { + uint32_t pos = (uint32_t)rep->iblk; + uint32_t off; + + for (k = 0, off = 0; k < blocks; ++k, off += rep->bs, ++pos) { + for (j = 0; j < (rep->bs - 3); j += 4) + sg_put_unaligned_be32(pos, rep->buffp + off + j); + } + } else if (clp->in_flags.zero) + memset(rep->buffp, 0, blocks * rep->bs); + else if (clp->in_flags.ff) + memset(rep->buffp, 0xff, blocks * rep->bs); + else { + for (k = 0, bp = rep->buffp; k < blocks; ++k, bp += rep->bs) { + for (j = 0; j < rep->bs; j += jbump) { + /* mrand48 takes uniformly from [-2^31, 2^31) */ +#ifdef HAVE_SRAND48_R + mrand48_r(&rep->drand, &rn); +#else + rn = mrand48(); +#endif + *((uint32_t *)(bp + j)) = (uint32_t)rn; + } + } + } + clp->in_rem_count -= blocks; + return stop_after_write; + } + if (! same_fds) { /* each has own file pointer, so we need to move it */ + int64_t pos = rep->iblk * rep->bs; + + if (lseek64(rep->infd, pos, SEEK_SET) < 0) { /* problem if pipe! */ + pr2serr_lk("%s: tid=%d: >> lseek64(%" PRId64 "): %s\n", __func__, + rep->id, pos, safe_strerror(errno)); + stop_both(clp); + return true; + } + } + /* enters holding in_mutex */ + while (((res = read(clp->infd, rep->buffp, blocks * rep->bs)) < 0) && + ((EINTR == errno) || (EAGAIN == errno))) + std::this_thread::yield();/* another thread may be able to progress */ + if (res < 0) { + char strerr_buff[STRERR_BUFF_LEN + 1]; + + if (clp->in_flags.coe) { + memset(rep->buffp, 0, rep->num_blks * rep->bs); + pr2serr_lk("tid=%d: >> substituted zeros for in blk=%" PRId64 + " for %d bytes, %s\n", rep->id, rep->iblk, + rep->num_blks * rep->bs, + tsafe_strerror(errno, strerr_buff)); + res = rep->num_blks * rep->bs; + } + else { + pr2serr_lk("tid=%d: error in normal read, %s\n", rep->id, + tsafe_strerror(errno, strerr_buff)); + stop_both(clp); + return true; + } + } + if (res < blocks * rep->bs) { + // int o_blocks = blocks; + + stop_after_write = true; + blocks = res / rep->bs; + if ((res % rep->bs) > 0) { + blocks++; + clp->in_partial++; + } + /* Reverse out + re-apply blocks on clp */ + // clp->in_blk -= o_blocks; + // clp->in_count += o_blocks; + rep->num_blks = blocks; + // clp->in_blk += blocks; + // clp->in_count -= blocks; + } + clp->in_rem_count -= blocks; + return stop_after_write; +} + +static void +normal_out_wr(Rq_elem * rep, int blocks) +{ + int res; + struct global_collection * clp = rep->clp; + + /* enters holding out_mutex */ + if (clp->verbose > 4) + pr2serr_lk("%s: tid=%d: oblk=%" PRIu64 ", blocks=%d\n", __func__, + rep->id, rep->oblk, blocks); + while (((res = write(clp->outfd, rep->buffp, blocks * rep->bs)) + < 0) && ((EINTR == errno) || (EAGAIN == errno))) + std::this_thread::yield();/* another thread may be able to progress */ + if (res < 0) { + char strerr_buff[STRERR_BUFF_LEN + 1]; + + if (clp->out_flags.coe) { + pr2serr_lk("tid=%d: >> ignored error for out blk=%" PRId64 + " for %d bytes, %s\n", rep->id, rep->oblk, + rep->num_blks * rep->bs, + tsafe_strerror(errno, strerr_buff)); + res = rep->num_blks * rep->bs; + } + else { + pr2serr_lk("tid=%d: error normal write, %s\n", rep->id, + tsafe_strerror(errno, strerr_buff)); + stop_both(clp); + return; + } + } + if (res < blocks * rep->bs) { + blocks = res / rep->bs; + if ((res % rep->bs) > 0) { + blocks++; + clp->out_partial++; + } + rep->num_blks = blocks; + } + clp->out_rem_count -= blocks; +} + +static int +sg_build_scsi_cdb(uint8_t * cdbp, int cdb_sz, unsigned int blocks, + int64_t start_block, bool ver_true, bool write_true, + bool fua, bool dpo) +{ + int rd_opcode[] = {0x8, 0x28, 0xa8, 0x88}; + int ve_opcode[] = {0xff /* no VER(6) */, 0x2f, 0xaf, 0x8f}; + int wr_opcode[] = {0xa, 0x2a, 0xaa, 0x8a}; + int sz_ind; + + memset(cdbp, 0, cdb_sz); + if (ver_true) { /* only support VERIFY(10) */ + if (cdb_sz < 10) { + pr2serr_lk("%sonly support VERIFY(10)\n", my_name); + return 1; + } + cdb_sz = 10; + fua = false; + cdbp[1] |= 0x2; /* BYTCHK=1 --> sending dout for comparison */ + cdbp[0] = ve_opcode[1]; + } + if (dpo) + cdbp[1] |= 0x10; + if (fua) + cdbp[1] |= 0x8; + switch (cdb_sz) { + case 6: + sz_ind = 0; + cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] : + rd_opcode[sz_ind]); + sg_put_unaligned_be24(0x1fffff & start_block, cdbp + 1); + cdbp[4] = (256 == blocks) ? 0 : (uint8_t)blocks; + if (blocks > 256) { + pr2serr_lk("%sfor 6 byte commands, maximum number of blocks is " + "256\n", my_name); + return 1; + } + if ((start_block + blocks - 1) & (~0x1fffff)) { + pr2serr_lk("%sfor 6 byte commands, can't address blocks beyond " + "%d\n", my_name, 0x1fffff); + return 1; + } + if (dpo || fua) { + pr2serr_lk("%sfor 6 byte commands, neither dpo nor fua bits " + "supported\n", my_name); + return 1; + } + break; + case 10: + if (! ver_true) { + sz_ind = 1; + cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] : + rd_opcode[sz_ind]); + } + sg_put_unaligned_be32((uint32_t)start_block, cdbp + 2); + sg_put_unaligned_be16((uint16_t)blocks, cdbp + 7); + if (blocks & (~0xffff)) { + pr2serr_lk("%sfor 10 byte commands, maximum number of blocks is " + "%d\n", my_name, 0xffff); + return 1; + } + break; + case 12: + sz_ind = 2; + cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] : + rd_opcode[sz_ind]); + sg_put_unaligned_be32((uint32_t)start_block, cdbp + 2); + sg_put_unaligned_be32((uint32_t)blocks, cdbp + 6); + break; + case 16: + sz_ind = 3; + cdbp[0] = (uint8_t)(write_true ? wr_opcode[sz_ind] : + rd_opcode[sz_ind]); + sg_put_unaligned_be64((uint64_t)start_block, cdbp + 2); + sg_put_unaligned_be32((uint32_t)blocks, cdbp + 10); + break; + default: + pr2serr_lk("%sexpected cdb size of 6, 10, 12, or 16 but got %d\n", + my_name, cdb_sz); + return 1; + } + return 0; +} + +/* Enters this function holding in_mutex */ +static void +sg_in_rd_cmd(struct global_collection * clp, Rq_elem * rep, + mrq_arr_t & def_arr) +{ + int status, pack_id; + + while (1) { + int res = sg_start_io(rep, def_arr, pack_id, NULL); + + if (1 == res) + err_exit(ENOMEM, "sg starting in command"); + else if (res < 0) { + pr2serr_lk("tid=%d: inputting to sg failed, blk=%" PRId64 "\n", + rep->id, rep->iblk); + status = pthread_mutex_unlock(&clp->in_mutex); + if (0 != status) err_exit(status, "unlock in_mutex"); + stop_both(clp); + return; + } + /* Now release in mutex to let other reads run in parallel */ + status = pthread_mutex_unlock(&clp->in_mutex); + if (0 != status) err_exit(status, "unlock in_mutex"); + + res = sg_finish_io(rep->wr, rep, pack_id, NULL); + switch (res) { + case SG_LIB_CAT_ABORTED_COMMAND: + case SG_LIB_CAT_UNIT_ATTENTION: + /* try again with same addr, count info */ + /* now re-acquire in mutex for balance */ + /* N.B. This re-read could now be out of read sequence */ + status = pthread_mutex_lock(&clp->in_mutex); + if (0 != status) err_exit(status, "lock in_mutex"); + break; /* will loop again */ + case SG_LIB_CAT_MEDIUM_HARD: + if (0 == clp->in_flags.coe) { + pr2serr_lk("error finishing sg in command (medium)\n"); + if (exit_status <= 0) + exit_status = res; + stop_both(clp); + return; + } else { + memset(get_buffp(rep), 0, rep->num_blks * rep->bs); + pr2serr_lk("tid=%d: >> substituted zeros for in blk=%" PRId64 + " for %d bytes\n", rep->id, rep->iblk, + rep->num_blks * rep->bs); + } +#if defined(__GNUC__) +#if (__GNUC__ >= 7) + __attribute__((fallthrough)); + /* FALL THROUGH */ +#endif +#endif + case 0: + status = pthread_mutex_lock(&clp->in_mutex); + if (0 != status) err_exit(status, "lock in_mutex"); + if (rep->dio_incomplete_count || rep->resid) { + clp->dio_incomplete_count += rep->dio_incomplete_count; + clp->sum_of_resids += rep->resid; + } + clp->in_rem_count -= rep->num_blks; + status = pthread_mutex_unlock(&clp->in_mutex); + if (0 != status) err_exit(status, "unlock in_mutex"); + return; + default: + pr2serr_lk("tid=%d: error finishing sg in command (%d)\n", + rep->id, res); + if (exit_status <= 0) + exit_status = res; + stop_both(clp); + return; + } + } /* end of while (1) loop */ +} + +static bool +sg_wr_swap_share(Rq_elem * rep, int to_fd, bool before) +{ + bool not_first = false; + int err = 0; + int k; + int read_side_fd = rep->infd; + struct global_collection * clp = rep->clp; + struct sg_extended_info sei {}; + struct sg_extended_info * seip = &sei; + + if (rep->clp->verbose > 2) + pr2serr_lk("%s: tid=%d: to_fd=%d, before=%d\n", __func__, rep->id, + to_fd, (int)before); + seip->sei_wr_mask |= SG_SEIM_CHG_SHARE_FD; + seip->sei_rd_mask |= SG_SEIM_CHG_SHARE_FD; + seip->share_fd = to_fd; + if (before) { + /* clear READ_SIDE_FINI bit: puts read-side in SG_RQ_SHR_SWAP state */ + seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS; + seip->sei_rd_mask |= SG_SEIM_CTL_FLAGS; + seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_READ_SIDE_FINI; + seip->ctl_flags &= ~SG_CTL_FLAGM_READ_SIDE_FINI;/* would be 0 anyway */ + } + for (k = 0; (ioctl(read_side_fd, SG_SET_GET_EXTENDED, seip) < 0) && + (EBUSY == errno); ++k) { + err = errno; + if (k > 10000) + break; + if (! not_first) { + if (clp->verbose > 3) + pr2serr_lk("tid=%d: ioctl(EXTENDED(change_shared_fd=%d), " + "failed errno=%d %s\n", rep->id, read_side_fd, err, + strerror(err)); + not_first = true; + } + err = 0; + std::this_thread::yield();/* another thread may be able to progress */ + } + if (err) { + pr2serr_lk("tid=%d: ioctl(EXTENDED(change_shared_fd=%d), failed " + "errno=%d %s\n", rep->id, read_side_fd, err, strerror(err)); + return false; + } + if (clp->verbose > 15) + pr2serr_lk("%s: tid=%d: ioctl(EXTENDED(change_shared_fd)) ok, " + "read_side_fd=%d, to_write_side_fd=%d\n", __func__, rep->id, + read_side_fd, to_fd); + return true; +} + +/* Enters this function holding out_mutex */ +static void +sg_out_wr_cmd(Rq_elem * rep, mrq_arr_t & def_arr, bool is_wr2, bool prefetch) +{ + int res, status, pack_id, nblks; + struct global_collection * clp = rep->clp; + uint32_t ofsplit = clp->ofsplit; + pthread_mutex_t * mutexp = is_wr2 ? &clp->out2_mutex : &clp->out_mutex; + struct sg_io_extra xtr {}; + struct sg_io_extra * xtrp = &xtr; + const char * wr_or_ver = clp->verify ? "verify" : "out"; + + xtrp->is_wr2 = is_wr2; + xtrp->prefetch = prefetch; + nblks = rep->num_blks; + if (rep->has_share && is_wr2) + sg_wr_swap_share(rep, rep->out2fd, true); + + if (prefetch) { +again: + res = sg_start_io(rep, def_arr, pack_id, xtrp); + if (1 == res) + err_exit(ENOMEM, "sg starting out command"); + else if (res < 0) { + pr2serr_lk("%ssg %s failed, blk=%" PRId64 "\n", + my_name, wr_or_ver, rep->oblk); + status = pthread_mutex_unlock(mutexp); + if (0 != status) err_exit(status, "unlock out_mutex"); + stop_both(clp); + goto fini; + } + /* Now release in mutex to let other reads run in parallel */ + status = pthread_mutex_unlock(mutexp); + if (0 != status) err_exit(status, "unlock out_mutex"); + + res = sg_finish_io(rep->wr, rep, pack_id, xtrp); + switch (res) { + case SG_LIB_CAT_ABORTED_COMMAND: + case SG_LIB_CAT_UNIT_ATTENTION: + /* try again with same addr, count info */ + /* now re-acquire out mutex for balance */ + /* N.B. This re-write could now be out of write sequence */ + status = pthread_mutex_lock(mutexp); + if (0 != status) err_exit(status, "lock out_mutex"); + goto again; + case SG_LIB_CAT_CONDITION_MET: + case 0: + status = pthread_mutex_lock(mutexp); + if (0 != status) err_exit(status, "unlock out_mutex"); + break; + default: + pr2serr_lk("error finishing sg prefetch command (%d)\n", res); + if (exit_status <= 0) + exit_status = res; + stop_both(clp); + goto fini; + } + } + + /* start write (or verify) on current segment on sg device */ + xtrp->prefetch = false; + if ((ofsplit > 0) && (rep->num_blks > (int)ofsplit)) { + xtrp->dout_is_split = true; + xtrp->blk_offset = 0; + xtrp->blks = ofsplit; + nblks = ofsplit; + xtrp->hpv4_ind = 0; + } +split_upper: + while (1) { + res = sg_start_io(rep, def_arr, pack_id, xtrp); + if (1 == res) + err_exit(ENOMEM, "sg starting out command"); + else if (res < 0) { + pr2serr_lk("%ssg %s failed, blk=%" PRId64 "\n", my_name, + wr_or_ver, rep->oblk); + status = pthread_mutex_unlock(mutexp); + if (0 != status) err_exit(status, "unlock out_mutex"); + stop_both(clp); + goto fini; + } + /* Now release in mutex to let other reads run in parallel */ + status = pthread_mutex_unlock(mutexp); + if (0 != status) err_exit(status, "unlock out_mutex"); + + res = sg_finish_io(rep->wr, rep, pack_id, xtrp); + switch (res) { + case SG_LIB_CAT_ABORTED_COMMAND: + case SG_LIB_CAT_UNIT_ATTENTION: + /* try again with same addr, count info */ + /* now re-acquire out mutex for balance */ + /* N.B. This re-write could now be out of write sequence */ + status = pthread_mutex_lock(mutexp); + if (0 != status) err_exit(status, "lock out_mutex"); + break; /* loops around */ + case SG_LIB_CAT_MEDIUM_HARD: + if (0 == clp->out_flags.coe) { + pr2serr_lk("error finishing sg %s command (medium)\n", + wr_or_ver); + if (exit_status <= 0) + exit_status = res; + stop_both(clp); + goto fini; + } else + pr2serr_lk(">> ignored error for %s blk=%" PRId64 " for %d " + "bytes\n", wr_or_ver, rep->oblk, nblks * rep->bs); +#if defined(__GNUC__) +#if (__GNUC__ >= 7) + __attribute__((fallthrough)); + /* FALL THROUGH */ +#endif +#endif + case SG_LIB_CAT_CONDITION_MET: + case 0: + if (! is_wr2) { + status = pthread_mutex_lock(mutexp); + if (0 != status) err_exit(status, "lock out_mutex"); + if (rep->dio_incomplete_count || rep->resid) { + clp->dio_incomplete_count += rep->dio_incomplete_count; + clp->sum_of_resids += rep->resid; + } + clp->out_rem_count -= nblks; + status = pthread_mutex_unlock(mutexp); + if (0 != status) err_exit(status, "unlock out_mutex"); + } + goto fini; + case SG_LIB_CAT_MISCOMPARE: + ++num_miscompare; + // fall through + default: + pr2serr_lk("error finishing sg %s command (%d)\n", wr_or_ver, + res); + if (exit_status <= 0) + exit_status = res; + stop_both(clp); + goto fini; + } + } /* end of while (1) loop */ +fini: + if (xtrp->dout_is_split) { /* set up upper half of split */ + if ((0 == xtrp->hpv4_ind) && (rep->num_blks > (int)ofsplit)) { + xtrp->hpv4_ind = 1; + xtrp->blk_offset = ofsplit; + xtrp->blks = rep->num_blks - ofsplit; + nblks = xtrp->blks; + status = pthread_mutex_lock(mutexp); + if (0 != status) err_exit(status, "lock out_mutex"); + goto split_upper; + } + } + if (rep->has_share && is_wr2) + sg_wr_swap_share(rep, rep->outfd, false); +} + +static int +process_mrq_response(Rq_elem * rep, const struct sg_io_v4 * ctl_v4p, + const struct sg_io_v4 * a_v4p, int num_mrq, + uint32_t & good_inblks, uint32_t & good_outblks) +{ + struct global_collection * clp = rep->clp; + bool ok, all_good; + bool sb_in_co = !!(ctl_v4p->response); + int id = rep->id; + int resid = ctl_v4p->din_resid; + int sres = ctl_v4p->spare_out; + int n_subm = num_mrq - ctl_v4p->dout_resid; + int n_cmpl = ctl_v4p->info; + int n_good = 0; + int hole_count = 0; + int vb = clp->verbose; + int k, j, f1, slen, blen; + char b[160]; + + blen = sizeof(b); + good_inblks = 0; + good_outblks = 0; + if (vb > 2) + pr2serr_lk("[thread_id=%d] %s: num_mrq=%d, n_subm=%d, n_cmpl=%d\n", + id, __func__, num_mrq, n_subm, n_cmpl); + if (n_subm < 0) { + pr2serr_lk("[%d] co.dout_resid(%d) > num_mrq(%d)\n", id, + ctl_v4p->dout_resid, num_mrq); + return -1; + } + if (n_cmpl != (num_mrq - resid)) + pr2serr_lk("[%d] co.info(%d) != (num_mrq(%d) - co.din_resid(%d))\n" + "will use co.info\n", id, n_cmpl, num_mrq, resid); + if (n_cmpl > n_subm) { + pr2serr_lk("[%d] n_cmpl(%d) > n_subm(%d), use n_subm for both\n", + id, n_cmpl, n_subm); + n_cmpl = n_subm; + } + if (sres) { + pr2serr_lk("[%d] secondary error: %s [%d], info=0x%x\n", id, + strerror(sres), sres, ctl_v4p->info); + if (E2BIG == sres) { + sg_take_snap(rep->infd, id, true); + sg_take_snap(rep->outfd, id, true); + } + } + /* Check if those submitted have finished or not. N.B. If there has been + * an error then there may be "holes" (i.e. info=0x0) in the array due + * to completions being out-of-order. */ + for (k = 0, j = 0; ((k < num_mrq) && (j < n_subm)); + ++k, j += f1, ++a_v4p) { + slen = a_v4p->response_len; + if (! (SG_INFO_MRQ_FINI & a_v4p->info)) + ++hole_count; + ok = true; + f1 = !!(a_v4p->info); /* want to skip n_subm count if info is 0x0 */ + if (SG_INFO_CHECK & a_v4p->info) { + ok = false; + pr2serr_lk("[%d] a_v4[%d]: SG_INFO_CHECK set [%s]\n", id, k, + sg_info_str(a_v4p->info, sizeof(b), b)); + } + if (sg_scsi_status_is_bad(a_v4p->device_status) || + a_v4p->transport_status || a_v4p->driver_status) { + ok = false; + if (SAM_STAT_CHECK_CONDITION != a_v4p->device_status) { + pr2serr_lk("[%d] a_v4[%d]:\n", id, k); + if (vb) + lk_chk_n_print4(" >>", a_v4p, vb > 4); + } + } + if (slen > 0) { + struct sg_scsi_sense_hdr ssh; + const uint8_t *sbp = (const uint8_t *) + (sb_in_co ? ctl_v4p->response : a_v4p->response); + + if (sg_scsi_normalize_sense(sbp, slen, &ssh) && + (ssh.response_code >= 0x70)) { + if (ssh.response_code & 0x1) + ok = true; + if (vb) { + sg_get_sense_str(" ", sbp, slen, false, blen, b); + pr2serr_lk("[%d] a_v4[%d]:\n%s\n", id, k, b); + } + } + } + if (ok && f1) { + ++n_good; + if (a_v4p->dout_xfer_len >= (uint32_t)rep->bs) + good_outblks += (a_v4p->dout_xfer_len - a_v4p->dout_resid) / + rep->bs; + if (a_v4p->din_xfer_len >= (uint32_t)rep->bs) + good_inblks += (a_v4p->din_xfer_len - a_v4p->din_resid) / + rep->bs; + } + } /* end of request array scan loop */ + if ((n_subm == num_mrq) || (vb < 3)) + goto fini; + pr2serr_lk("[%d] checking response array _beyond_ number of " + "submissions [%d] to num_mrq:\n", id, k); + for (all_good = true; k < num_mrq; ++k, ++a_v4p) { + if (SG_INFO_MRQ_FINI & a_v4p->info) { + pr2serr_lk("[%d] a_v4[%d]: unexpected SG_INFO_MRQ_FINI set [%s]\n", + id, k, sg_info_str(a_v4p->info, sizeof(b), b)); + all_good = false; + } + if (a_v4p->device_status || a_v4p->transport_status || + a_v4p->driver_status) { + pr2serr_lk("[%d] a_v4[%d]:\n", id, k); + lk_chk_n_print4(" ", a_v4p, vb > 4); + all_good = false; + } + } + if (all_good) + pr2serr_lk(" ... all good\n"); +fini: + return n_good; +} + +#if 0 +static int +chk_mrq_response(Rq_elem * rep, const struct sg_io_v4 * ctl_v4p, + const struct sg_io_v4 * a_v4p, int nrq, + uint32_t * good_inblksp, uint32_t * good_outblksp) +{ + struct global_collection * clp = rep->clp; + bool ok; + int id = rep->id; + int resid = ctl_v4p->din_resid; + int sres = ctl_v4p->spare_out; + int n_subm = nrq - ctl_v4p->dout_resid; + int n_cmpl = ctl_v4p->info; + int n_good = 0; + int vb = clp->verbose; + int k, slen, blen; + uint32_t good_inblks = 0; + uint32_t good_outblks = 0; + const struct sg_io_v4 * a_np = a_v4p; + char b[80]; + + blen = sizeof(b); + if (n_subm < 0) { + pr2serr_lk("[%d] %s: co.dout_resid(%d) > nrq(%d)\n", id, __func__, + ctl_v4p->dout_resid, nrq); + return -1; + } + if (n_cmpl != (nrq - resid)) + pr2serr_lk("[%d] %s: co.info(%d) != (nrq(%d) - co.din_resid(%d))\n" + "will use co.info\n", id, __func__, n_cmpl, nrq, resid); + if (n_cmpl > n_subm) { + pr2serr_lk("[%d] %s: n_cmpl(%d) > n_subm(%d), use n_subm for both\n", + id, __func__, n_cmpl, n_subm); + n_cmpl = n_subm; + } + if (sres) { + pr2serr_lk("[%d] %s: secondary error: %s [%d], info=0x%x\n", id, + __func__, strerror(sres), sres, ctl_v4p->info); + if (E2BIG == sres) { + sg_take_snap(rep->infd, id, true); + sg_take_snap(rep->outfd, id, true); + } + } + /* Check if those submitted have finished or not */ + for (k = 0; k < n_subm; ++k, ++a_np) { + slen = a_np->response_len; + if (! (SG_INFO_MRQ_FINI & a_np->info)) { + pr2serr_lk("[%d] %s, a_n[%d]: missing SG_INFO_MRQ_FINI [%s]\n", + id, __func__, k, sg_info_str(a_np->info, blen, b)); + v4hdr_out_lk("a_np", a_np, id); + v4hdr_out_lk("cop", ctl_v4p, id); + } + ok = true; + if (SG_INFO_CHECK & a_np->info) { + ok = false; + pr2serr_lk("[%d] a_n[%d]: SG_INFO_CHECK set [%s]\n", id, k, + sg_info_str(a_np->info, sizeof(b), b)); + } + if (sg_scsi_status_is_bad(a_np->device_status) || + a_np->transport_status || a_np->driver_status) { + ok = false; + if (SAM_STAT_CHECK_CONDITION != a_np->device_status) { + pr2serr_lk("[%d] %s, a_n[%d]:\n", id, __func__, k); + if (vb) + lk_chk_n_print4(" >>", a_np, false); + } + } + if (slen > 0) { + struct sg_scsi_sense_hdr ssh; + const uint8_t *sbp = (const uint8_t *)a_np->response; + + if (sg_scsi_normalize_sense(sbp, slen, &ssh) && + (ssh.response_code >= 0x70)) { + char b[256]; + + if (ssh.response_code & 0x1) + ok = true; + if (vb) { + sg_get_sense_str(" ", sbp, slen, false, sizeof(b), b); + pr2serr_lk("[%d] %s, a_n[%d]:\n%s\n", id, __func__, k, b); + } + } + } + if (ok) { + ++n_good; + if (a_np->dout_xfer_len >= (uint32_t)rep->bs) + good_outblks += (a_np->dout_xfer_len - a_np->dout_resid) / + rep->bs; + if (a_np->din_xfer_len >= (uint32_t)rep->bs) + good_inblks += (a_np->din_xfer_len - a_np->din_resid) / + rep->bs; + } + } + if ((n_subm == nrq) || (vb < 3)) + goto fini; + pr2serr_lk("[%d] %s: checking response array beyond number of " + "submissions:\n", id, __func__); + for (k = n_subm; k < nrq; ++k, ++a_np) { + if (SG_INFO_MRQ_FINI & a_np->info) + pr2serr_lk("[%d] %s, a_n[%d]: unexpected SG_INFO_MRQ_FINI set\n", + id, __func__, k); + if (a_np->device_status || a_np->transport_status || + a_np->driver_status) { + pr2serr_lk("[%d] %s, a_n[%d]:\n", id, __func__, k); + lk_chk_n_print4(" ", a_np, vb > 4); + } + } +fini: + if (good_inblksp) + *good_inblksp = good_inblks; + if (good_outblksp) + *good_outblksp = good_outblks; + return n_good; +} +#endif + +/* do mrq 'full non-blocking' invocation so both submission and completion + * is async (i.e. uses SGV4_FLAG_IMMED flag). This type of mrq is + * restricted to a single file descriptor (i.e. the 'fd' argument). */ +static int +sgh_do_async_mrq(Rq_elem * rep, mrq_arr_t & def_arr, int fd, + struct sg_io_v4 * ctlop, int nrq) +{ + int half = nrq / 2; + int k, res, nwait, half_num, rest, err, num_good, b_len; + const int64_t wait_us = 10; + uint32_t in_fin_blks, out_fin_blks; + struct sg_io_v4 * a_v4p; + struct sg_io_v4 hold_ctlo; + struct global_collection * clp = rep->clp; + char b[80]; + + hold_ctlo = *ctlop; + b_len = sizeof(b); + a_v4p = def_arr.first.data(); + ctlop->flags = SGV4_FLAG_MULTIPLE_REQS; + if (clp->in_flags.polled || clp->out_flags.polled) { + /* submit of full non-blocking with POLLED */ + ctlop->flags |= (SGV4_FLAG_IMMED | SGV4_FLAG_POLLED); + if (!after1 && (clp->verbose > 1)) { + after1 = true; + pr2serr_lk("%s: %s\n", __func__, mrq_s_nb_s); + } + } else { + ctlop->flags |= SGV4_FLAG_IMMED; /* submit non-blocking */ + if (!after1 && (clp->verbose > 1)) { + after1 = true; + pr2serr_lk("%s: %s\n", __func__, mrq_s_nb_s); + } + } + if (clp->verbose > 4) { + pr2serr_lk("%s: Controlling object _before_ ioctl(SG_IOSUBMIT):\n", + __func__); + if (clp->verbose > 5) + hex2stderr_lk((const uint8_t *)ctlop, sizeof(*ctlop), 1); + v4hdr_out_lk("Controlling object before", ctlop, rep->id); + } + res = ioctl(fd, SG_IOSUBMIT, ctlop); + if (res < 0) { + err = errno; + if (E2BIG == err) + sg_take_snap(fd, rep->id, true); + pr2serr_lk("%s: ioctl(SG_IOSUBMIT, %s)-->%d, errno=%d: %s\n", __func__, + sg_flags_str(ctlop->flags, b_len, b), res, err, + strerror(err)); + return -1; + } + /* fetch first half */ + for (k = 0; k < 100000; ++k) { + ++num_waiting_calls; + res = ioctl(fd, SG_GET_NUM_WAITING, &nwait); + if (res < 0) { + err = errno; + pr2serr_lk("%s: ioctl(SG_GET_NUM_WAITING)-->%d, errno=%d: %s\n", + __func__, res, err, strerror(err)); + return -1; + } + if (nwait >= half) + break; + this_thread::sleep_for(chrono::microseconds{wait_us}); + } + ctlop->flags = (SGV4_FLAG_MULTIPLE_REQS | SGV4_FLAG_IMMED); + res = ioctl(fd, SG_IORECEIVE, ctlop); + if (res < 0) { + err = errno; + if (ENODATA != err) { + pr2serr_lk("%s: ioctl(SG_IORECEIVE, %s),1-->%d, errno=%d: %s\n", + __func__, sg_flags_str(ctlop->flags, b_len, b), res, + err, strerror(err)); + return -1; + } + half_num = 0; + } else + half_num = ctlop->info; + if (clp->verbose > 4) { + pr2serr_lk("%s: Controlling object output by ioctl(SG_IORECEIVE),1: " + "num_received=%d\n", __func__, half_num); + if (clp->verbose > 5) + hex2stderr_lk((const uint8_t *)ctlop, sizeof(*ctlop), 1); + v4hdr_out_lk("Controlling object after", ctlop, rep->id); + if (clp->verbose > 5) { + for (k = 0; k < half_num; ++k) { + pr2serr_lk("AFTER: def_arr[%d]:\n", k); + v4hdr_out_lk("normal v4 object", (a_v4p + k), rep->id); + // hex2stderr_lk((const uint8_t *)(a_v4p + k), sizeof(*a_v4p), + // 1); + } + } + } + in_fin_blks = 0; + out_fin_blks = 0; + num_good = process_mrq_response(rep, ctlop, a_v4p, half_num, in_fin_blks, + out_fin_blks); + if (clp->verbose > 2) + pr2serr_lk("%s: >>>1 num_good=%d, in_q/fin blks=%u/%u; out_q/fin " + "blks=%u/%u\n", __func__, num_good, rep->in_mrq_q_blks, + in_fin_blks, rep->out_mrq_q_blks, out_fin_blks); + + if (num_good < 0) + res = -1; + else if (num_good < half_num) { + int resid_blks = rep->in_mrq_q_blks - in_fin_blks; + + if (resid_blks > 0) + gcoll.in_rem_count += resid_blks; + resid_blks = rep->out_mrq_q_blks - out_fin_blks; + if (resid_blks > 0) + gcoll.out_rem_count += resid_blks; + + return -1; + } + + rest = nrq - half_num; + if (rest < 1) + goto fini; + /* fetch remaining */ + for (k = 0; k < 100000; ++k) { + ++num_waiting_calls; + res = ioctl(fd, SG_GET_NUM_WAITING, &nwait); + if (res < 0) { + pr2serr_lk("%s: ioctl(SG_GET_NUM_WAITING)-->%d, errno=%d: %s\n", + __func__, res, errno, strerror(errno)); + return -1; + } + if (nwait >= rest) + break; + this_thread::sleep_for(chrono::microseconds{wait_us}); + } + ctlop = &hold_ctlo; + ctlop->din_xferp += (half_num * sizeof(struct sg_io_v4)); + ctlop->din_xfer_len -= (half_num * sizeof(struct sg_io_v4)); + ctlop->dout_xferp = ctlop->din_xferp; + ctlop->dout_xfer_len = ctlop->din_xfer_len; + ctlop->flags = (SGV4_FLAG_MULTIPLE_REQS | SGV4_FLAG_IMMED); + res = ioctl(fd, SG_IORECEIVE, ctlop); + if (res < 0) { + err = errno; + if (ENODATA != err) { + pr2serr_lk("%s: ioctl(SG_IORECEIVE, %s),2-->%d, errno=%d: %s\n", + __func__, sg_flags_str(ctlop->flags, b_len, b), res, + err, strerror(err)); + return -1; + } + half_num = 0; + } else + half_num = ctlop->info; + if (clp->verbose > 4) { + pr2serr_lk("%s: Controlling object output by ioctl(SG_IORECEIVE),2: " + "num_received=%d\n", __func__, half_num); + if (clp->verbose > 5) + hex2stderr_lk((const uint8_t *)ctlop, sizeof(*ctlop), 1); + v4hdr_out_lk("Controlling object after", ctlop, rep->id); + if (clp->verbose > 5) { + for (k = 0; k < half_num; ++k) { + pr2serr_lk("AFTER: def_arr[%d]:\n", k); + v4hdr_out_lk("normal v4 object", (a_v4p + k), rep->id); + // hex2stderr_lk((const uint8_t *)(a_v4p + k), sizeof(*a_v4p), + // 1); + } + } + } + in_fin_blks = 0; + out_fin_blks = 0; + num_good = process_mrq_response(rep, ctlop, a_v4p, half_num, in_fin_blks, + out_fin_blks); + if (clp->verbose > 2) + pr2serr_lk("%s: >>>2 num_good=%d, in_q/fin blks=%u/%u; out_q/fin " + "blks=%u/%u\n", __func__, num_good, rep->in_mrq_q_blks, + in_fin_blks, rep->out_mrq_q_blks, out_fin_blks); + + if (num_good < 0) + res = -1; + else if (num_good < half_num) { + int resid_blks = rep->in_mrq_q_blks - in_fin_blks; + + if (resid_blks > 0) + gcoll.in_rem_count += resid_blks; + resid_blks = rep->out_mrq_q_blks - out_fin_blks; + if (resid_blks > 0) + gcoll.out_rem_count += resid_blks; + + res = -1; + } + +fini: + return res; +} + +/* Split def_arr into fd_def_arr and o_fd_arr based on whether each element's + * flags field has SGV4_FLAG_DO_ON_OTHER set. If it is set place in + * o_fd_def_arr and mask out SGV4_DO_ON_OTHER. Returns number of elements + * in o_fd_def_arr. */ +static int +split_def_arr(const mrq_arr_t & def_arr, mrq_arr_t & fd_def_arr, + mrq_arr_t & o_fd_def_arr) +{ + int nrq, k; + int res = 0; + const struct sg_io_v4 * a_v4p; + + a_v4p = def_arr.first.data(); + nrq = def_arr.first.size(); + + for (k = 0; k < nrq; ++k) { + int flags; + const struct sg_io_v4 * h4p = a_v4p + k; + + flags = h4p->flags; + if (flags & SGV4_FLAG_DO_ON_OTHER) { + o_fd_def_arr.first.push_back(def_arr.first[k]); + o_fd_def_arr.second.push_back(def_arr.second[k]); + flags &= ~SGV4_FLAG_DO_ON_OTHER; /* mask out DO_ON_OTHER */ + o_fd_def_arr.first[res].flags = flags; + ++res; + } else { + fd_def_arr.first.push_back(def_arr.first[k]); + fd_def_arr.second.push_back(def_arr.second[k]); + } + } + return res; +} + +/* This function sets up a multiple request (mrq) transaction and sends it + * to the pass-through. Returns 0 on success, 1 if ENOMEM error else -1 for + * other errors. */ +static int +sgh_do_deferred_mrq(Rq_elem * rep, mrq_arr_t & def_arr) +{ + bool launch_mrq_abort = false; + int nrq, k, res, fd, mrq_pack_id, status, id, num_good, b_len; + uint32_t in_fin_blks, out_fin_blks; + const int max_cdb_sz = 16; + struct sg_io_v4 * a_v4p; + struct sg_io_v4 ctl_v4 {}; + uint8_t * cmd_ap = NULL; + struct global_collection * clp = rep->clp; + const char * iosub_str = "iosub_str"; + char b[80]; + + id = rep->id; + b_len = sizeof(b); + ctl_v4.guard = 'Q'; + a_v4p = def_arr.first.data(); + nrq = def_arr.first.size(); + if (nrq < 1) { + pr2serr_lk("[%d] %s: strange nrq=0, nothing to do\n", id, __func__); + return 0; + } + if (clp->mrq_cmds) { + cmd_ap = (uint8_t *)calloc(nrq, max_cdb_sz); + if (NULL == cmd_ap) { + pr2serr_lk("[%d] %s: no memory for calloc(%d * 16)\n", id, + __func__, nrq); + return 1; + } + } + for (k = 0; k < nrq; ++k) { + struct sg_io_v4 * h4p = a_v4p + k; + uint8_t *cmdp = &def_arr.second[k].front(); + + if (clp->mrq_cmds) { + memcpy(cmd_ap + (k * max_cdb_sz), cmdp, h4p->request_len); + h4p->request = 0; + } else + h4p->request = (uint64_t)cmdp; + if (clp->verbose > 5) { + pr2serr_lk("%s%s[%d] def_arr[%d]", ((0 == k) ? __func__ : ""), + ((0 == k) ? ": " : ""), id, k); + if (h4p->din_xferp) + pr2serr_lk(" [din=0x%p]:\n", (void *)h4p->din_xferp); + else if (h4p->dout_xferp) + pr2serr_lk(" [dout=0x%p]:\n", (void *)h4p->dout_xferp); + else + pr2serr_lk(":\n"); + hex2stderr_lk((const uint8_t *)h4p, sizeof(*h4p), 1); + } + } + if (rep->both_sg || rep->same_sg) + fd = rep->infd; /* assume share to rep->outfd */ + else if (rep->only_in_sg) + fd = rep->infd; + else if (rep->only_out_sg) + fd = rep->outfd; + else { + pr2serr_lk("[%d] %s: why am I here? No sg devices\n", id, __func__); + res = -1; + goto fini; + } + res = 0; + if (clp->mrq_cmds) { + ctl_v4.request_len = nrq * max_cdb_sz; + ctl_v4.request = (uint64_t)cmd_ap; + } + ctl_v4.flags = SGV4_FLAG_MULTIPLE_REQS; + if (! clp->mrq_async) { + ctl_v4.flags |= SGV4_FLAG_STOP_IF; + if (clp->in_flags.mrq_svb || clp->out_flags.mrq_svb) + ctl_v4.flags |= SGV4_FLAG_SHARE; + } + ctl_v4.dout_xferp = (uint64_t)a_v4p; /* request array */ + ctl_v4.dout_xfer_len = nrq * sizeof(*a_v4p); + ctl_v4.din_xferp = (uint64_t)a_v4p; /* response array */ + ctl_v4.din_xfer_len = nrq * sizeof(*a_v4p); + mrq_pack_id = atomic_fetch_add(&mono_mrq_id, 1); + if ((clp->m_aen > 0) && (MONO_MRQ_ID_INIT != mrq_pack_id) && + (0 == ((mrq_pack_id - MONO_MRQ_ID_INIT) % clp->m_aen))) { + launch_mrq_abort = true; + if (clp->verbose > 2) + pr2serr_lk("[%d] %s: Decide to launch MRQ abort thread, " + "mrq_id=%d\n", id, __func__, mrq_pack_id); + memset(&rep->mai, 0, sizeof(rep->mai)); + rep->mai.from_tid = id; + rep->mai.mrq_id = mrq_pack_id; + rep->mai.fd = fd; + rep->mai.debug = clp->verbose; + + status = pthread_create(&rep->mrq_abort_thread_id, NULL, + mrq_abort_thread, (void *)&rep->mai); + if (0 != status) err_exit(status, "pthread_create, sig..."); + } + ctl_v4.request_extra = launch_mrq_abort ? mrq_pack_id : 0; + rep->mrq_id = mrq_pack_id; + if (clp->verbose && rep->both_sg && clp->mrq_async) + iosub_str = "SG_IOSUBMIT(variable)"; + if (clp->verbose > 4) { + pr2serr_lk("%s: Controlling object _before_ ioctl(%s):\n", + __func__, iosub_str); + if (clp->verbose > 5) + hex2stderr_lk((const uint8_t *)&ctl_v4, sizeof(ctl_v4), 1); + v4hdr_out_lk("Controlling object before", &ctl_v4, id); + } + if (clp->mrq_async && (! rep->both_sg)) { + /* do 'submit non-blocking' or 'full non-blocking' mrq */ + mrq_arr_t fd_def_arr; + mrq_arr_t o_fd_def_arr; + + /* need to deconstruct def_arr[] into two separate lists, one for + * the source, the other for the destination. */ + int o_num_fd = split_def_arr(def_arr, fd_def_arr, o_fd_def_arr); + int num_fd = fd_def_arr.first.size(); + if (num_fd > 0) { + struct sg_io_v4 fd_ctl = ctl_v4; + struct sg_io_v4 * aa_v4p = fd_def_arr.first.data(); + + for (k = 0; k < num_fd; ++k) { + struct sg_io_v4 * h4p = aa_v4p + k; + uint8_t *cmdp = &fd_def_arr.second[k].front(); + + if (clp->mrq_cmds) { + memcpy(cmd_ap + (k * max_cdb_sz), cmdp, h4p->request_len); + h4p->request = 0; + } else + h4p->request = (uint64_t)cmdp; + if (clp->verbose > 5) { + pr2serr_lk("[%d] df_def_arr[%d]:\n", id, k); + hex2stderr_lk((const uint8_t *)(aa_v4p + k), + sizeof(*aa_v4p), 1); + } + } + fd_ctl.dout_xferp = (uint64_t)aa_v4p; /* request array */ + fd_ctl.dout_xfer_len = num_fd * sizeof(*aa_v4p); + fd_ctl.din_xferp = (uint64_t)aa_v4p; /* response array */ + fd_ctl.din_xfer_len = num_fd * sizeof(*aa_v4p); + fd_ctl.request_extra = launch_mrq_abort ? mrq_pack_id : 0; + /* this is the source side mrq command */ + res = sgh_do_async_mrq(rep, fd_def_arr, fd, &fd_ctl, num_fd); + rep->in_mrq_q_blks = 0; + if (res) + goto fini; + } + if (o_num_fd > 0) { + struct sg_io_v4 o_fd_ctl = ctl_v4; + struct sg_io_v4 * aa_v4p = o_fd_def_arr.first.data(); + + for (k = 0; k < o_num_fd; ++k) { + struct sg_io_v4 * h4p = aa_v4p + k; + uint8_t *cmdp = &o_fd_def_arr.second[k].front(); + + if (clp->mrq_cmds) { + memcpy(cmd_ap + (k * max_cdb_sz), cmdp, h4p->request_len); + h4p->request = 0; + } else + h4p->request = (uint64_t)cmdp; + if (clp->verbose > 5) { + pr2serr_lk("[%d] o_fd_def_arr[%d]:\n", id, k); + hex2stderr_lk((const uint8_t *)(aa_v4p + k), + sizeof(*aa_v4p), 1); + } + } + o_fd_ctl.dout_xferp = (uint64_t)aa_v4p; /* request array */ + o_fd_ctl.dout_xfer_len = o_num_fd * sizeof(*aa_v4p); + o_fd_ctl.din_xferp = (uint64_t)aa_v4p; /* response array */ + o_fd_ctl.din_xfer_len = o_num_fd * sizeof(*aa_v4p); + o_fd_ctl.request_extra = launch_mrq_abort ? mrq_pack_id : 0; + /* this is the destination side mrq command */ + res = sgh_do_async_mrq(rep, o_fd_def_arr, rep->outfd, &o_fd_ctl, + o_num_fd); + rep->out_mrq_q_blks = 0; + } + goto fini; + } + +try_again: + if (clp->unbalanced_mrq) { + iosub_str = "SG_IOSUBMIT(variable_blocking)"; + if (!after1 && (clp->verbose > 1)) { + after1 = true; + pr2serr_lk("%s: unbalanced %s\n", __func__, mrq_vb_s); + } + res = ioctl(fd, SG_IOSUBMIT, &ctl_v4); + } else { + if (clp->mrq_async) { + iosub_str = "SG_IOSUBMIT(variable_blocking)"; + if (!after1 && (clp->verbose > 1)) { + after1 = true; + pr2serr_lk("%s: %s\n", __func__, mrq_vb_s); + } + res = ioctl(fd, SG_IOSUBMIT, &ctl_v4); + } else if (clp->in_flags.mrq_svb || clp->out_flags.mrq_svb) { + iosub_str = "SG_IOSUBMIT(shared_variable_blocking)"; + if (!after1 && (clp->verbose > 1)) { + after1 = true; + pr2serr_lk("%s: %s\n", __func__, mrq_svb_s); + } + res = ioctl(fd, SG_IOSUBMIT, &ctl_v4); + } else { + iosub_str = "SG_IO(ordered_blocking)"; + if (!after1 && (clp->verbose > 1)) { + after1 = true; + pr2serr_lk("%s: %s\n", __func__, mrq_blk_s); + } + res = ioctl(fd, SG_IO, &ctl_v4); + } + } + if (res < 0) { + int err = errno; + + if (E2BIG == err) + sg_take_snap(fd, id, true); + else if (EBUSY == err) { + ++num_ebusy; + std::this_thread::yield();/* allow another thread to progress */ + goto try_again; + } + pr2serr_lk("%s: ioctl(%s, %s)-->%d, errno=%d: %s\n", + __func__, iosub_str, sg_flags_str(ctl_v4.flags, b_len, b), + res, err, strerror(err)); + res = -1; + goto fini; + } + if (clp->verbose && vb_first_time.load()) { + pr2serr_lk("First controlling object output by ioctl(%s), flags: " + "%s\n", iosub_str, sg_flags_str(ctl_v4.flags, b_len, b)); + vb_first_time.store(false); + } else if (clp->verbose > 4) + pr2serr_lk("%s: Controlling object output by ioctl(%s):\n", + __func__, iosub_str); + if (clp->verbose > 4) { + if (clp->verbose > 5) + hex2stderr_lk((const uint8_t *)&ctl_v4, sizeof(ctl_v4), 1); + v4hdr_out_lk("Controlling object after", &ctl_v4, id); + if (clp->verbose > 5) { + for (k = 0; k < nrq; ++k) { + pr2serr_lk("AFTER: def_arr[%d]:\n", k); + v4hdr_out_lk("normal v4 object", (a_v4p + k), id); + // hex2stderr_lk((const uint8_t *)(a_v4p + k), sizeof(*a_v4p), + // 1); + } + } + } + in_fin_blks = 0; + out_fin_blks = 0; + num_good = process_mrq_response(rep, &ctl_v4, a_v4p, nrq, in_fin_blks, + out_fin_blks); + if (clp->verbose > 2) + pr2serr_lk("%s: >>> num_good=%d, in_q/fin blks=%u/%u; out_q/fin " + "blks=%u/%u\n", __func__, num_good, rep->in_mrq_q_blks, + in_fin_blks, rep->out_mrq_q_blks, out_fin_blks); + + if (num_good < 0) + res = -1; + else if (num_good < nrq) { + int resid_blks = rep->in_mrq_q_blks - in_fin_blks; + + if (resid_blks > 0) + gcoll.in_rem_count += resid_blks; + resid_blks = rep->out_mrq_q_blks - out_fin_blks; + if (resid_blks > 0) + gcoll.out_rem_count += resid_blks; + + res = -1; + } + rep->in_mrq_q_blks = 0; + rep->out_mrq_q_blks = 0; +fini: + def_arr.first.clear(); + def_arr.second.clear(); + if (cmd_ap) + free(cmd_ap); + if (launch_mrq_abort) { + if (clp->verbose > 1) + pr2serr_lk("[%d] %s: About to join MRQ abort thread, " + "mrq_id=%d\n", id, __func__, mrq_pack_id); + + void * vp; /* not used */ + status = pthread_join(rep->mrq_abort_thread_id, &vp); + if (0 != status) err_exit(status, "pthread_join"); + } + return res; +} + +/* Returns 0 on success, 1 if ENOMEM error else -1 for other errors. */ +static int +sg_start_io(Rq_elem * rep, mrq_arr_t & def_arr, int & pack_id, + struct sg_io_extra *xtrp) +{ + struct global_collection * clp = rep->clp; + bool wr = rep->wr; + bool fua = wr ? clp->out_flags.fua : clp->in_flags.fua; + bool dpo = wr ? clp->out_flags.dpo : clp->in_flags.dpo; + bool dio = wr ? clp->out_flags.dio : clp->in_flags.dio; + bool mmap = wr ? clp->out_flags.mmap : clp->in_flags.mmap; + bool noxfer = wr ? clp->out_flags.noxfer : clp->in_flags.noxfer; + bool v4 = wr ? clp->out_flags.v4 : clp->in_flags.v4; + bool qhead = wr ? clp->out_flags.qhead : clp->in_flags.qhead; + bool qtail = wr ? clp->out_flags.qtail : clp->in_flags.qtail; + bool polled = wr ? clp->out_flags.polled : clp->in_flags.polled; + bool mout_if = wr ? clp->out_flags.mout_if : clp->in_flags.mout_if; + bool prefetch = xtrp ? xtrp->prefetch : false; + bool is_wr2 = xtrp ? xtrp->is_wr2 : false; + int cdbsz = wr ? clp->cdbsz_out : clp->cdbsz_in; + int flags = 0; + int res, err, fd, b_len, nblks, blk_off; + int64_t blk = wr ? rep->oblk : rep->iblk; + struct sg_io_hdr * hp = &rep->io_hdr; + struct sg_io_v4 * h4p = &rep->io_hdr4[xtrp ? xtrp->hpv4_ind : 0]; + const char * cp = ""; + const char * crwp; + char b[80]; + + b_len = sizeof(b); + if (wr) { + fd = is_wr2 ? rep->out2fd : rep->outfd; + if (clp->verify) { + crwp = is_wr2 ? "verifying2" : "verifying"; + if (prefetch) + crwp = is_wr2 ? "prefetch2" : "prefetch"; + } else + crwp = is_wr2 ? "writing2" : "writing"; + } else { + fd = rep->infd; + crwp = "reading"; + } + if (qhead) + qtail = false; /* qhead takes precedence */ + + if (v4 && xtrp && xtrp->dout_is_split) { + res = sg_build_scsi_cdb(rep->cmd, cdbsz, xtrp->blks, + blk + (unsigned int)xtrp->blk_offset, + clp->verify, true, fua, dpo); + } else + res = sg_build_scsi_cdb(rep->cmd, cdbsz, rep->num_blks, blk, + wr ? clp->verify : false, wr, fua, dpo); + if (res) { + pr2serr_lk("%sbad cdb build, start_blk=%" PRId64 ", blocks=%d\n", + my_name, blk, rep->num_blks); + return -1; + } + if (prefetch) { + if (cdbsz == 10) + rep->cmd[0] = SGP_PRE_FETCH10; + else if (cdbsz == 16) + rep->cmd[0] = SGP_PRE_FETCH16; + else { + pr2serr_lk("%sbad PRE-FETCH build, start_blk=%" PRId64 ", " + "blocks=%d\n", my_name, blk, rep->num_blks); + return -1; + } + rep->cmd[1] = 0x2; /* set IMMED (no fua or dpo) */ + } + if (mmap && (clp->noshare || (rep->outregfd >= 0))) + flags |= SG_FLAG_MMAP_IO; + if (noxfer) + flags |= SG_FLAG_NO_DXFER; + if (dio) + flags |= SG_FLAG_DIRECT_IO; + if (polled) + flags |= SGV4_FLAG_POLLED; + if (qhead) + flags |= SG_FLAG_Q_AT_HEAD; + if (qtail) + flags |= SG_FLAG_Q_AT_TAIL; + if (mout_if) + flags |= SGV4_FLAG_META_OUT_IF; + if (rep->has_share) { + flags |= SGV4_FLAG_SHARE; + if (wr) + flags |= SGV4_FLAG_NO_DXFER; + else if (rep->outregfd < 0) + flags |= SGV4_FLAG_NO_DXFER; + + cp = (wr ? " write_side active" : " read_side active"); + } else + cp = (wr ? " write-side not sharing" : " read_side not sharing"); + if (rep->both_sg) { + if (wr) + pack_id = rep->rd_p_id + 1; + else { + pack_id = 2 * atomic_fetch_add(&mono_pack_id, 1); + rep->rd_p_id = pack_id; + } + } else + pack_id = atomic_fetch_add(&mono_pack_id, 1); /* fetch before */ + rep->rq_id = pack_id; + nblks = rep->num_blks; + blk_off = 0; + if (clp->verbose && 0 == clp->nmrqs && vb_first_time.load()) { + vb_first_time.store(false); + pr2serr("First normal IO: %s, flags: %s\n", cp, + sg_flags_str(flags, b_len, b)); + } + if (v4) { + memset(h4p, 0, sizeof(struct sg_io_v4)); + if (clp->nmrqs > 0) { + if (rep->both_sg && (rep->outfd == fd)) + flags |= SGV4_FLAG_DO_ON_OTHER; + } + if (xtrp && xtrp->dout_is_split && (nblks > 0)) { + if (1 == xtrp->hpv4_ind) { + flags |= SGV4_FLAG_DOUT_OFFSET; + blk_off = xtrp->blk_offset; + h4p->spare_in = clp->bs * blk_off; + } + nblks = xtrp->blks; + if ((0 == xtrp->hpv4_ind) && (nblks < rep->num_blks)) + flags |= SGV4_FLAG_KEEP_SHARE; + } + if (clp->ofile2_given && wr && rep->has_share && ! is_wr2) + flags |= SGV4_FLAG_KEEP_SHARE; /* set on first write only */ + else if (clp->fail_mask & 1) + flags |= SGV4_FLAG_KEEP_SHARE; /* troublemaking .... */ + } else + memset(hp, 0, sizeof(struct sg_io_hdr)); + if (clp->verbose > 3) { + bool lock = true; + char prefix[128]; + + if (4 == clp->verbose) { + snprintf(prefix, sizeof(prefix), "tid,rq_id=%d,%d: ", rep->id, + pack_id); + lock = false; + } else { + prefix[0] = '\0'; + pr2serr_lk("%s tid,rq_id=%d,%d: SCSI %s%s %s, blk=%" PRId64 + " num_blks=%d\n", __func__, rep->id, pack_id, crwp, cp, + sg_flags_str(flags, b_len, b), blk + blk_off, nblks); + } + lk_print_command_len(prefix, rep->cmd, cdbsz, lock); + } + if (v4) + goto do_v4; // <<<<<<<<<<<<<<< look further down + + hp->interface_id = 'S'; + hp->cmd_len = cdbsz; + hp->cmdp = rep->cmd; + hp->dxferp = get_buffp(rep); + hp->dxfer_len = clp->bs * rep->num_blks; + if (!wr) + hp->dxfer_direction = SG_DXFER_FROM_DEV; + else if (prefetch) { + hp->dxfer_direction = SG_DXFER_NONE; + hp->dxfer_len = 0; + hp->dxferp = NULL; + } else + hp->dxfer_direction = SG_DXFER_TO_DEV; + hp->mx_sb_len = sizeof(rep->sb); + hp->sbp = rep->sb; + hp->timeout = clp->cmd_timeout; + hp->usr_ptr = rep; + hp->pack_id = pack_id; + hp->flags = flags; + + while (((res = write(fd, hp, sizeof(struct sg_io_hdr))) < 0) && + ((EINTR == errno) || (EAGAIN == errno) || (EBUSY == errno))) { + if (EAGAIN == errno) { + ++num_start_eagain; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } else if (EBUSY == errno) { + ++num_ebusy; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } + std::this_thread::yield();/* another thread may be able to progress */ + } + err = errno; + if (res < 0) { + if (ENOMEM == err) + return 1; + pr2serr_lk("%s tid=%d: %s %s write(2) failed: %s\n", __func__, + rep->id, cp, sg_flags_str(hp->flags, b_len, b), + strerror(err)); + return -1; + } + return 0; + +do_v4: + h4p->guard = 'Q'; + h4p->request_len = cdbsz; + h4p->request = (uint64_t)rep->cmd; + if (wr) { + if (prefetch) { + h4p->dout_xfer_len = 0; // din_xfer_len is also 0 + h4p->dout_xferp = 0; + } else { + h4p->dout_xfer_len = clp->bs * nblks; + h4p->dout_xferp = (uint64_t)get_buffp(rep); + } + } else if (nblks > 0) { + h4p->din_xfer_len = clp->bs * nblks; + h4p->din_xferp = (uint64_t)get_buffp(rep); + } + h4p->max_response_len = sizeof(rep->sb); + h4p->response = (uint64_t)rep->sb; + h4p->timeout = clp->cmd_timeout; + h4p->usr_ptr = (uint64_t)rep; + h4p->request_extra = pack_id; /* this is the pack_id */ + h4p->flags = flags; + if (clp->nmrqs > 0) { + big_cdb cdb_arr; + uint8_t * cmdp = &(cdb_arr[0]); + + if (wr) + rep->out_mrq_q_blks += nblks; + else + rep->in_mrq_q_blks += nblks; + memcpy(cmdp, rep->cmd, cdbsz); + def_arr.first.push_back(*h4p); + def_arr.second.push_back(cdb_arr); + res = 0; + if ((int)def_arr.first.size() >= clp->nmrqs) { + res = sgh_do_deferred_mrq(rep, def_arr); + if (res) + pr2serr_lk("%s tid=%d: sgh_do_deferred_mrq failed\n", + __func__, rep->id); + } + return res; + } + while (((res = ioctl(fd, SG_IOSUBMIT, h4p)) < 0) && + ((EINTR == errno) || (EAGAIN == errno) || (EBUSY == errno))) { + if (EAGAIN == errno) { + ++num_start_eagain; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } else if (EBUSY == errno) { + ++num_ebusy; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } + std::this_thread::yield();/* another thread may be able to progress */ + } + err = errno; + if (res < 0) { + if (ENOMEM == err) + return 1; + if (E2BIG == err) + sg_take_snap(fd, rep->id, true); + pr2serr_lk("%s tid=%d: %s %s ioctl(2) failed: %s\n", __func__, + rep->id, cp, sg_flags_str(h4p->flags, b_len, b), + strerror(err)); + // v4hdr_out_lk("leadin", h4p, rep->id); + return -1; + } + if ((clp->aen > 0) && (rep->rep_count > 0)) { + if (0 == (rep->rq_id % clp->aen)) { + struct timespec tspec = {0, 4000 /* 4 usecs */}; + + nanosleep(&tspec, NULL); +#if 0 + struct pollfd a_poll; + + a_poll.fd = fd; + a_poll.events = POLL_IN; + a_poll.revents = 0; + res = poll(&a_poll, 1 /* element */, 1 /* millisecond */); + if (res < 0) + pr2serr_lk("%s: poll() failed: %s [%d]\n", + __func__, safe_strerror(errno), errno); + else if (0 == res) { /* timeout, cmd still inflight, so abort */ + } +#endif + ++num_abort_req; + res = ioctl(fd, SG_IOABORT, h4p); + if (res < 0) { + err = errno; + if (ENODATA == err) { + if (clp->verbose > 2) + pr2serr_lk("%s: ioctl(SG_IOABORT) no match on " + "pack_id=%d\n", __func__, pack_id); + } else + pr2serr_lk("%s: ioctl(SG_IOABORT) failed: %s [%d]\n", + __func__, safe_strerror(err), err); + } else { + ++num_abort_req_success; + if (clp->verbose > 2) + pr2serr_lk("%s: sent ioctl(SG_IOABORT) on rq_id=%d, " + "success\n", __func__, pack_id); + } + } /* else got response, too late for timeout, so skip */ + } + return 0; +} + +/* 0 -> successful, SG_LIB_CAT_UNIT_ATTENTION or SG_LIB_CAT_ABORTED_COMMAND + -> try again, SG_LIB_CAT_NOT_READY, SG_LIB_CAT_MEDIUM_HARD, + -1 other errors */ +static int +sg_finish_io(bool wr, Rq_elem * rep, int pack_id, struct sg_io_extra *xtrp) +{ + struct global_collection * clp = rep->clp; + bool v4 = wr ? clp->out_flags.v4 : clp->in_flags.v4; + bool mout_if = wr ? clp->out_flags.mout_if : clp->in_flags.mout_if; + bool is_wr2 = xtrp ? xtrp->is_wr2 : false; + bool prefetch = xtrp ? xtrp->prefetch : false; + int res, fd; + int64_t blk = wr ? rep->oblk : rep->iblk; + struct sg_io_hdr io_hdr; + struct sg_io_hdr * hp; + struct sg_io_v4 * h4p; + const char *cp; + + if (wr) { + fd = is_wr2 ? rep->out2fd : rep->outfd; + cp = is_wr2 ? "writing2" : "writing"; + if (clp->verify) { + cp = is_wr2 ? "verifying2" : "verifying"; + if (prefetch) + cp = is_wr2 ? "prefetch2" : "prefetch"; + } + } else { + fd = rep->infd; + cp = "reading"; + } + if (v4) + goto do_v4; + memset(&io_hdr, 0 , sizeof(struct sg_io_hdr)); + /* FORCE_PACK_ID active set only read packet with matching pack_id */ + io_hdr.interface_id = 'S'; + io_hdr.dxfer_direction = wr ? SG_DXFER_TO_DEV : SG_DXFER_FROM_DEV; + io_hdr.pack_id = pack_id; + + while (((res = read(fd, &io_hdr, sizeof(struct sg_io_hdr))) < 0) && + ((EINTR == errno) || (EAGAIN == errno) || (EBUSY == errno))) { + if (EAGAIN == errno) { + ++num_fin_eagain; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } else if (EBUSY == errno) { + ++num_ebusy; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } + std::this_thread::yield();/* another thread may be able to progress */ + } + if (res < 0) { + perror("finishing io [read(2)] on sg device, error"); + return -1; + } + if (rep != (Rq_elem *)io_hdr.usr_ptr) + err_exit(0, "sg_finish_io: bad usr_ptr, request-response mismatch\n"); + memcpy(&rep->io_hdr, &io_hdr, sizeof(struct sg_io_hdr)); + hp = &rep->io_hdr; + + res = sg_err_category3(hp); + switch (res) { + case SG_LIB_CAT_CLEAN: + case SG_LIB_CAT_CONDITION_MET: + break; + case SG_LIB_CAT_RECOVERED: + lk_chk_n_print3(cp, hp, false); + break; + case SG_LIB_CAT_ABORTED_COMMAND: + case SG_LIB_CAT_UNIT_ATTENTION: + if (clp->verbose > 3) + lk_chk_n_print3(cp, hp, false); + return res; + case SG_LIB_CAT_MISCOMPARE: + ++num_miscompare; + // fall through + case SG_LIB_CAT_NOT_READY: + default: + { + char ebuff[EBUFF_SZ]; + + snprintf(ebuff, EBUFF_SZ, "%s blk=%" PRId64, cp, blk); + lk_chk_n_print3(ebuff, hp, clp->verbose > 1); + return res; + } + } + if ((wr ? clp->out_flags.dio : clp->in_flags.dio) && + (! (hp->info & SG_INFO_DIRECT_IO_MASK))) + rep->dio_incomplete_count = 1; /* count dios done as indirect IO */ + else + rep->dio_incomplete_count = 0; + rep->resid = hp->resid; + if (clp->verbose > 3) + pr2serr_lk("%s: tid=%d: completed %s\n", __func__, rep->id, cp); + return 0; + +do_v4: + if (clp->nmrqs > 0) { + rep->resid = 0; + return 0; + } + h4p = &rep->io_hdr4[xtrp ? xtrp->hpv4_ind : 0]; + h4p->request_extra = pack_id; + if (mout_if) { + h4p->info = 0; + h4p->din_resid = 0; + } + while (((res = ioctl(fd, SG_IORECEIVE, h4p)) < 0) && + ((EINTR == errno) || (EAGAIN == errno) || (EBUSY == errno))) { + if (EAGAIN == errno) { + ++num_fin_eagain; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } else if (EBUSY == errno) { + ++num_ebusy; +#ifdef SGH_DD_SNAP_DEV + if (0 == (num_ebusy % 1000)) + sg_take_snap(fd, rep->id, (clp->verbose > 2)); +#endif + } + std::this_thread::yield();/* another thread may be able to progress */ + } + if (res < 0) { + perror("finishing io [SG_IORECEIVE] on sg device, error"); + return -1; + } + if (mout_if && (0 == h4p->info) && (0 == h4p->din_resid)) + goto all_good; + if (rep != (Rq_elem *)h4p->usr_ptr) + err_exit(0, "sg_finish_io: bad usr_ptr, request-response mismatch\n"); + res = sg_err_category_new(h4p->device_status, h4p->transport_status, + h4p->driver_status, + (const uint8_t *)h4p->response, + h4p->response_len); + switch (res) { + case SG_LIB_CAT_CLEAN: + case SG_LIB_CAT_CONDITION_MET: + break; + case SG_LIB_CAT_RECOVERED: + lk_chk_n_print4(cp, h4p, false); + break; + case SG_LIB_CAT_ABORTED_COMMAND: + case SG_LIB_CAT_UNIT_ATTENTION: + if (clp->verbose > 3) + lk_chk_n_print4(cp, h4p, false); + return res; + case SG_LIB_CAT_MISCOMPARE: + ++num_miscompare; + // fall through + case SG_LIB_CAT_NOT_READY: + default: + { + char ebuff[EBUFF_SZ]; + + snprintf(ebuff, EBUFF_SZ, "%s rq_id=%d, blk=%" PRId64, cp, + pack_id, blk); + lk_chk_n_print4(ebuff, h4p, clp->verbose > 1); + if ((clp->verbose > 4) && h4p->info) + pr2serr_lk(" info=0x%x sg_info_check=%d direct=%d " + "detaching=%d aborted=%d\n", h4p->info, + !!(h4p->info & SG_INFO_CHECK), + !!(h4p->info & SG_INFO_DIRECT_IO), + !!(h4p->info & SG_INFO_DEVICE_DETACHING), + !!(h4p->info & SG_INFO_ABORTED)); + return res; + } + } + if ((wr ? clp->out_flags.dio : clp->in_flags.dio) && + ! (h4p->info & SG_INFO_DIRECT_IO)) + rep->dio_incomplete_count = 1; /* count dios done as indirect IO */ + else + rep->dio_incomplete_count = 0; + rep->resid = h4p->din_resid; + if (clp->verbose > 4) { + pr2serr_lk("%s: tid,rq_id=%d,%d: completed %s\n", __func__, rep->id, + pack_id, cp); + if ((clp->verbose > 4) && h4p->info) + pr2serr_lk(" info=0x%x sg_info_check=%d direct=%d " + "detaching=%d aborted=%d\n", h4p->info, + !!(h4p->info & SG_INFO_CHECK), + !!(h4p->info & SG_INFO_DIRECT_IO), + !!(h4p->info & SG_INFO_DEVICE_DETACHING), + !!(h4p->info & SG_INFO_ABORTED)); + } +all_good: + return 0; +} + +/* Returns reserved_buffer_size/mmap_size if success, else 0 for failure */ +static int +sg_prepare_resbuf(int fd, bool is_in, struct global_collection *clp, + uint8_t **mmpp) +{ + static bool done = false; + bool def_res = is_in ? clp->in_flags.defres : clp->out_flags.defres; + bool no_dur = is_in ? clp->in_flags.no_dur : clp->out_flags.no_dur; + bool masync = is_in ? clp->in_flags.masync : clp->out_flags.masync; + bool wq_excl = is_in ? clp->in_flags.wq_excl : clp->out_flags.wq_excl; + bool skip_thresh = is_in ? clp->in_flags.no_thresh : + clp->out_flags.no_thresh; + int res, t; + int num = 0; + uint8_t *mmp; + struct sg_extended_info sei {}; + struct sg_extended_info * seip = &sei; + + res = ioctl(fd, SG_GET_VERSION_NUM, &t); + if ((res < 0) || (t < 40000)) { + if (ioctl(fd, SG_GET_RESERVED_SIZE, &num) < 0) { + perror("SG_GET_RESERVED_SIZE ioctl failed"); + return 0; + } + if (! done) { + done = true; + sg_version_lt_4 = true; + pr2serr_lk("%ssg driver prior to 4.0.00, reduced functionality\n", + my_name); + } + goto bypass; + } + if (! sg_version_ge_40045) + goto bypass; + if (clp->elem_sz >= 4096) { + seip->sei_rd_mask |= SG_SEIM_SGAT_ELEM_SZ; + res = ioctl(fd, SG_SET_GET_EXTENDED, seip); + if (res < 0) + pr2serr_lk("%s%s: SG_SET_GET_EXTENDED(SGAT_ELEM_SZ) rd " + "error: %s\n", my_name, __func__, strerror(errno)); + if (clp->elem_sz != (int)seip->sgat_elem_sz) { + memset(seip, 0, sizeof(*seip)); + seip->sei_wr_mask |= SG_SEIM_SGAT_ELEM_SZ; + seip->sgat_elem_sz = clp->elem_sz; + res = ioctl(fd, SG_SET_GET_EXTENDED, seip); + if (res < 0) + pr2serr_lk("%s%s: SG_SET_GET_EXTENDED(SGAT_ELEM_SZ) wr " + "error: %s\n", my_name, __func__, strerror(errno)); + } + } + if (no_dur || masync || skip_thresh) { + memset(seip, 0, sizeof(*seip)); + seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS; + if (no_dur) { + seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_NO_DURATION; + seip->ctl_flags |= SG_CTL_FLAGM_NO_DURATION; + } + if (masync) { + seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_MORE_ASYNC; + seip->ctl_flags |= SG_CTL_FLAGM_MORE_ASYNC; + } + if (wq_excl) { + seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_EXCL_WAITQ; + seip->ctl_flags |= SG_CTL_FLAGM_EXCL_WAITQ; + } + if (skip_thresh) { + seip->tot_fd_thresh = 0; + sei.sei_wr_mask |= SG_SEIM_TOT_FD_THRESH; + } + res = ioctl(fd, SG_SET_GET_EXTENDED, seip); + if (res < 0) + pr2serr_lk("%s%s: SG_SET_GET_EXTENDED(NO_DURATION) error: %s\n", + my_name, __func__, strerror(errno)); + } +bypass: + if (! def_res) { + num = clp->bs * clp->bpt; + res = ioctl(fd, SG_SET_RESERVED_SIZE, &num); + if (res < 0) { + perror("sgh_dd: SG_SET_RESERVED_SIZE error"); + return 0; + } else { + int nn; + + res = ioctl(fd, SG_GET_RESERVED_SIZE, &nn); + if (res < 0) { + perror("sgh_dd: SG_GET_RESERVED_SIZE error"); + return 0; + } + if (nn < num) { + pr2serr_lk("%s: SG_GET_RESERVED_SIZE shows size truncated, " + "wanted %d got %d\n", __func__, num, nn); + return 0; + } + } + if (mmpp) { + mmp = (uint8_t *)mmap(NULL, num, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (MAP_FAILED == mmp) { + int err = errno; + + pr2serr_lk("%s%s: sz=%d, fd=%d, mmap() failed: %s\n", + my_name, __func__, num, fd, strerror(err)); + return 0; + } + *mmpp = mmp; + } + } + t = 1; + res = ioctl(fd, SG_SET_FORCE_PACK_ID, &t); + if (res < 0) + perror("sgh_dd: SG_SET_FORCE_PACK_ID error"); + if (clp->unit_nanosec && sg_version_ge_40045) { + memset(seip, 0, sizeof(*seip)); + seip->sei_wr_mask |= SG_SEIM_CTL_FLAGS; + seip->ctl_flags_wr_mask |= SG_CTL_FLAGM_TIME_IN_NS; + seip->ctl_flags |= SG_CTL_FLAGM_TIME_IN_NS; + if (ioctl(fd, SG_SET_GET_EXTENDED, seip) < 0) { + pr2serr_lk("ioctl(EXTENDED(TIME_IN_NS)) failed, errno=%d %s\n", + errno, strerror(errno)); + } + } + t = 1; + res = ioctl(fd, SG_SET_DEBUG, &t); /* more info in the kernel log */ + if (res < 0) + perror("sgs_dd: SG_SET_DEBUG error"); + return (res < 0) ? 0 : num; +} + +static bool +process_flags(const char * arg, struct flags_t * fp) +{ + char buff[256]; + char * cp; + char * np; + + strncpy(buff, arg, sizeof(buff)); + buff[sizeof(buff) - 1] = '\0'; + if ('\0' == buff[0]) { + pr2serr("no flag found\n"); + return false; + } + cp = buff; + do { + np = strchr(cp, ','); + if (np) + *np++ = '\0'; + if (0 == strcmp(cp, "00")) + fp->zero = true; + else if (0 == strcmp(cp, "append")) + fp->append = true; + else if (0 == strcmp(cp, "coe")) + fp->coe = true; + else if (0 == strcmp(cp, "defres")) + fp->defres = true; + else if (0 == strcmp(cp, "dio")) + fp->dio = true; + else if (0 == strcmp(cp, "direct")) + fp->direct = true; + else if (0 == strcmp(cp, "dpo")) + fp->dpo = true; + else if (0 == strcmp(cp, "dsync")) + fp->dsync = true; + else if (0 == strcmp(cp, "excl")) + fp->excl = true; + else if (0 == strcmp(cp, "ff")) + fp->ff = true; + else if (0 == strcmp(cp, "fua")) + fp->fua = true; + else if (0 == strcmp(cp, "hipri")) + fp->polled = true; + else if (0 == strcmp(cp, "masync")) + fp->masync = true; + else if (0 == strcmp(cp, "mmap")) + ++fp->mmap; /* mmap > 1 stops munmap() being called */ + else if (0 == strcmp(cp, "mrq_imm")) + fp->mrq_immed = true; + else if (0 == strcmp(cp, "mrq_immed")) + fp->mrq_immed = true; + else if (0 == strcmp(cp, "mrq_svb")) + fp->mrq_svb = true; + else if (0 == strcmp(cp, "nodur")) + fp->no_dur = true; + else if (0 == strcmp(cp, "no_dur")) + fp->no_dur = true; + else if (0 == strcmp(cp, "nocreat")) + fp->nocreat = true; + else if (0 == strcmp(cp, "noshare")) + fp->noshare = true; + else if (0 == strcmp(cp, "no_share")) + fp->noshare = true; + else if (0 == strcmp(cp, "no_thresh")) + fp->no_thresh = true; + else if (0 == strcmp(cp, "no-thresh")) + fp->no_thresh = true; + else if (0 == strcmp(cp, "nothresh")) + fp->no_thresh = true; + else if (0 == strcmp(cp, "no_unshare")) + fp->no_unshare = true; + else if (0 == strcmp(cp, "no-unshare")) + fp->no_unshare = true; + else if (0 == strcmp(cp, "no_waitq")) + fp->no_waitq = true; + else if (0 == strcmp(cp, "no-waitq")) + fp->no_waitq = true; + else if (0 == strcmp(cp, "nowaitq")) + fp->no_waitq = true; + else if (0 == strcmp(cp, "noxfer")) + fp->noxfer = true; + else if (0 == strcmp(cp, "no_xfer")) + fp->noxfer = true; + else if (0 == strcmp(cp, "null")) + ; + else if (0 == strcmp(cp, "polled")) + fp->polled = true; + else if (0 == strcmp(cp, "qhead")) + fp->qhead = true; + else if (0 == strcmp(cp, "qtail")) + fp->qtail = true; + else if (0 == strcmp(cp, "random")) + fp->random = true; + else if ((0 == strcmp(cp, "mout_if")) || (0 == strcmp(cp, "mout-if"))) + fp->mout_if = true; + else if (0 == strcmp(cp, "same_fds")) + fp->same_fds = true; + else if (0 == strcmp(cp, "swait")) + fp->swait = true; + else if (0 == strcmp(cp, "v3")) + fp->v3 = true; + else if (0 == strcmp(cp, "v4")) { + fp->v4 = true; + fp->v4_given = true; + } else if (0 == strcmp(cp, "wq_excl")) + fp->wq_excl = true; + else { + pr2serr("unrecognised flag: %s\n", cp); + return false; + } + cp = np; + } while (cp); + return true; +} + +/* Returns the number of times 'ch' is found in string 's' given the + * string's length. */ +static int +num_chs_in_str(const char * s, int slen, int ch) +{ + int res = 0; + + while (--slen >= 0) { + if (ch == s[slen]) + ++res; + } + return res; +} + +static int +sg_in_open(struct global_collection *clp, const char *inf, uint8_t **mmpp, + int * mmap_lenp) +{ + int fd, n; + int flags = O_RDWR; + + if (clp->in_flags.direct) + flags |= O_DIRECT; + if (clp->in_flags.excl) + flags |= O_EXCL; + if (clp->in_flags.dsync) + flags |= O_SYNC; + + if ((fd = open(inf, flags)) < 0) { + int err = errno; + char ebuff[EBUFF_SZ]; + + snprintf(ebuff, EBUFF_SZ, "%s: could not open %s for sg reading", + __func__, inf); + perror(ebuff); + return -sg_convert_errno(err); + } + n = sg_prepare_resbuf(fd, true, clp, mmpp); + if (n <= 0) { + close(fd); + return -SG_LIB_FILE_ERROR; + } + if (clp->noshare) + sg_noshare_enlarge(fd, clp->verbose > 3); + if (mmap_lenp) + *mmap_lenp = n; + return fd; +} + +static int +sg_out_open(struct global_collection *clp, const char *outf, uint8_t **mmpp, + int * mmap_lenp) +{ + int fd, n; + int flags = O_RDWR; + + if (clp->out_flags.direct) + flags |= O_DIRECT; + if (clp->out_flags.excl) + flags |= O_EXCL; + if (clp->out_flags.dsync) + flags |= O_SYNC; + + if ((fd = open(outf, flags)) < 0) { + int err = errno; + char ebuff[EBUFF_SZ]; + + snprintf(ebuff, EBUFF_SZ, "%s: could not open %s for sg %s", + __func__, outf, (clp->verify ? "verifying" : "writing")); + perror(ebuff); + return -sg_convert_errno(err); + } + n = sg_prepare_resbuf(fd, false, clp, mmpp); + if (n <= 0) { + close(fd); + return -SG_LIB_FILE_ERROR; + } + if (clp->noshare) + sg_noshare_enlarge(fd, clp->verbose > 3); + if (mmap_lenp) + *mmap_lenp = n; + return fd; +} + +/* Process arguments given to 'conv=" option. Returns 0 on success, + * 1 on error. */ +static int +process_conv(const char * arg, struct flags_t * ifp, struct flags_t * ofp) +{ + char buff[256]; + char * cp; + char * np; + + strncpy(buff, arg, sizeof(buff)); + buff[sizeof(buff) - 1] = '\0'; + if ('\0' == buff[0]) { + pr2serr("no conversions found\n"); + return 1; + } + cp = buff; + do { + np = strchr(cp, ','); + if (np) + *np++ = '\0'; + if (0 == strcmp(cp, "nocreat")) + ofp->nocreat = true; + else if (0 == strcmp(cp, "noerror")) + ifp->coe = true; /* will still fail on write error */ + else if (0 == strcmp(cp, "notrunc")) + ; /* this is the default action of sg_dd so ignore */ + else if (0 == strcmp(cp, "null")) + ; + else if (0 == strcmp(cp, "sync")) + ; /* dd(susv4): pad errored block(s) with zeros but sg_dd does + * that by default. Typical dd use: 'conv=noerror,sync' */ + else { + pr2serr("unrecognised flag: %s\n", cp); + return 1; + } + cp = np; + } while (cp); + return 0; +} + +#define STR_SZ 1024 +#define INOUTF_SZ 512 + +static int +parse_cmdline_sanity(int argc, char * argv[], struct global_collection * clp, + char * inf, char * outf, char * out2f, char * outregf) +{ + bool verbose_given = false; + bool version_given = false; + bool verify_given = false; + bool bpt_given = false; + int ibs = 0; + int obs = 0; + int k, keylen, n, res; + char str[STR_SZ]; + char * key; + char * buf; + const char * cp; + + for (k = 1; k < argc; k++) { + if (argv[k]) { + strncpy(str, argv[k], STR_SZ); + str[STR_SZ - 1] = '\0'; + } + else + continue; + for (key = str, buf = key; *buf && *buf != '=';) + buf++; + if (*buf) + *buf++ = '\0'; + keylen = strlen(key); + if (0 == strcmp(key, "ae")) { + clp->aen = sg_get_num(buf); + if (clp->aen < 0) { + pr2serr("%sbad AEN argument to 'ae=', want 0 or higher\n", + my_name); + return SG_LIB_SYNTAX_ERROR; + } + cp = strchr(buf, ','); + if (cp) { + clp->m_aen = sg_get_num(cp + 1); + if (clp->m_aen < 0) { + pr2serr("%sbad MAEN argument to 'ae=', want 0 or " + "higher\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + clp->m_aen_given = true; + } + clp->aen_given = true; + } else if (0 == strcmp(key, "bpt")) { + clp->bpt = sg_get_num(buf); + if ((clp->bpt < 0) || (clp->bpt > MAX_BPT_VALUE)) { + pr2serr("%sbad argument to 'bpt='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + bpt_given = true; + } else if (0 == strcmp(key, "bs")) { + clp->bs = sg_get_num(buf); + if ((clp->bs < 0) || (clp->bs > MAX_BPT_VALUE)) { + pr2serr("%sbad argument to 'bs='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "cdbsz")) { + clp->cdbsz_in = sg_get_num(buf); + if ((clp->cdbsz_in < 6) || (clp->cdbsz_in > 32)) { + pr2serr("%s'cdbsz' expects 6, 10, 12, 16 or 32\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + clp->cdbsz_out = clp->cdbsz_in; + clp->cdbsz_given = true; + } else if (0 == strcmp(key, "coe")) { + clp->in_flags.coe = !! sg_get_num(buf); + clp->out_flags.coe = clp->in_flags.coe; + } else if (0 == strcmp(key, "conv")) { + if (process_conv(buf, &clp->in_flags, &clp->out_flags)) { + pr2serr("%s: bad argument to 'conv='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "count")) { + if (0 != strcmp("-1", buf)) { + dd_count = sg_get_llnum(buf); + if ((dd_count < 0) || (dd_count > MAX_COUNT_SKIP_SEEK)) { + pr2serr("%sbad argument to 'count='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } /* treat 'count=-1' as calculate count (same as not given) */ + } else if (0 == strcmp(key, "dio")) { + clp->in_flags.dio = !! sg_get_num(buf); + clp->out_flags.dio = clp->in_flags.dio; + } else if (0 == strcmp(key, "elemsz_kb")) { + n = sg_get_num(buf); + if (n < 1) { + pr2serr("elemsz_kb=EKB wants an integer > 0\n"); + return SG_LIB_SYNTAX_ERROR; + } + if (n & (n - 1)) { + pr2serr("elemsz_kb=EKB wants EKB to be power of 2\n"); + return SG_LIB_SYNTAX_ERROR; + } + clp->elem_sz = n * 1024; + } else if ((0 == strcmp(key, "fail_mask")) || + (0 == strcmp(key, "fail-mask"))) { + clp->fail_mask = sg_get_num(buf); + if (clp->fail_mask < 0) { + pr2serr("fail_mask: couldn't decode argument\n"); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "fua")) { + n = sg_get_num(buf); + if (n & 1) + clp->out_flags.fua = true; + if (n & 2) + clp->in_flags.fua = true; + } else if (0 == strcmp(key, "ibs")) { + ibs = sg_get_num(buf); + if ((ibs < 0) || (ibs > MAX_BPT_VALUE)) { + pr2serr("%sbad argument to 'ibs='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "if")) { + if ('\0' != inf[0]) { + pr2serr("Second 'if=' argument??\n"); + return SG_LIB_SYNTAX_ERROR; + } else { + memcpy(inf, buf, INOUTF_SZ); + inf[INOUTF_SZ - 1] = '\0'; /* noisy compiler */ + } + } else if (0 == strcmp(key, "iflag")) { + if (! process_flags(buf, &clp->in_flags)) { + pr2serr("%sbad argument to 'iflag='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "mrq")) { + if (isdigit(buf[0])) + cp = buf; + else { + if ('I' == isupper(buf[0])) + clp->is_mrq_i = true; + else if ('O' == isupper(buf[0])) + clp->is_mrq_o = true; + else { + pr2serr("%sonly mrq=i,NRQS or mrq=o,NRQS allowed here\n", + my_name); + return SG_LIB_SYNTAX_ERROR; + } + cp = strchr(buf, ','); + ++cp; + } + clp->nmrqs = sg_get_num(cp); + if (clp->nmrqs < 0) { + pr2serr("%sbad argument to 'mrq='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + cp = strchr(cp, ','); + if (cp && ('C' == toupper(cp[1]))) + clp->mrq_cmds = true; + } else if (0 == strcmp(key, "noshare")) { + clp->noshare = !! sg_get_num(buf); + } else if (0 == strcmp(key, "obs")) { + obs = sg_get_num(buf); + if ((obs < 0) || (obs > MAX_BPT_VALUE)) { + pr2serr("%sbad argument to 'obs='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (strcmp(key, "of2") == 0) { + if ('\0' != out2f[0]) { + pr2serr("Second OFILE2 argument??\n"); + return SG_LIB_CONTRADICT; + } else { + memcpy(out2f, buf, INOUTF_SZ); + out2f[INOUTF_SZ - 1] = '\0'; /* noisy compiler */ + } + } else if (strcmp(key, "ofreg") == 0) { + if ('\0' != outregf[0]) { + pr2serr("Second OFREG argument??\n"); + return SG_LIB_CONTRADICT; + } else { + memcpy(outregf, buf, INOUTF_SZ); + outregf[INOUTF_SZ - 1] = '\0'; /* noisy compiler */ + } + } else if (0 == strcmp(key, "ofsplit")) { + clp->ofsplit = sg_get_num(buf); + if (-1 == clp->ofsplit) { + pr2serr("%sbad argument to 'ofsplit='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (strcmp(key, "of") == 0) { + if ('\0' != outf[0]) { + pr2serr("Second 'of=' argument??\n"); + return SG_LIB_SYNTAX_ERROR; + } else { + memcpy(outf, buf, INOUTF_SZ); + outf[INOUTF_SZ - 1] = '\0'; /* noisy compiler */ + } + } else if (0 == strcmp(key, "oflag")) { + if (! process_flags(buf, &clp->out_flags)) { + pr2serr("%sbad argument to 'oflag='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "sdt")) { + cp = strchr(buf, ','); + n = sg_get_num(buf); + if (n < 0) { + pr2serr("%sbad argument to 'sdt=CRT[,ICT]'\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + clp->sdt_crt = n; + if (cp) { + n = sg_get_num(cp + 1); + if (n < 0) { + pr2serr("%sbad 2nd argument to 'sdt=CRT,ICT'\n", + my_name); + return SG_LIB_SYNTAX_ERROR; + } + clp->sdt_ict = n; + } + } else if (0 == strcmp(key, "seek")) { + clp->seek = sg_get_llnum(buf); + if (clp->seek < 0) { + pr2serr("%sbad argument to 'seek='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "skip")) { + clp->skip = sg_get_llnum(buf); + if (clp->skip < 0) { + pr2serr("%sbad argument to 'skip='\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strcmp(key, "sync")) + do_sync = !! sg_get_num(buf); + else if (0 == strcmp(key, "thr")) + num_threads = sg_get_num(buf); + else if (0 == strcmp(key, "time")) { + do_time = sg_get_num(buf); + if (do_time < 0) { + pr2serr("%sbad argument to 'time=0|1|2'\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + cp = strchr(buf, ','); + if (cp) { + n = sg_get_num(cp + 1); + if (n < 0) { + pr2serr("%sbad argument to 'time=0|1|2,TO'\n", my_name); + return SG_LIB_SYNTAX_ERROR; + } + clp->cmd_timeout = n ? (n * 1000) : DEF_TIMEOUT; + } + } else if (0 == strcmp(key, "unshare")) + clp->unshare = !! sg_get_num(buf); /* default: true */ + else if (0 == strncmp(key, "verb", 4)) + clp->verbose = sg_get_num(buf); + else if ((keylen > 1) && ('-' == key[0]) && ('-' != key[1])) { + res = 0; + n = num_chs_in_str(key + 1, keylen - 1, 'c'); + clp->chkaddr += n; + res += n; + n = num_chs_in_str(key + 1, keylen - 1, 'd'); + clp->dry_run += n; + res += n; + n = num_chs_in_str(key + 1, keylen - 1, 'h'); + clp->help += n; + res += n; + n = num_chs_in_str(key + 1, keylen - 1, 'p'); + if (n > 0) + clp->prefetch = true; + res += n; + n = num_chs_in_str(key + 1, keylen - 1, 'v'); + if (n > 0) + verbose_given = true; + clp->verbose += n; /* -v ---> --verbose */ + res += n; + n = num_chs_in_str(key + 1, keylen - 1, 'V'); + if (n > 0) + version_given = true; + res += n; + n = num_chs_in_str(key + 1, keylen - 1, 'x'); + if (n > 0) + verify_given = true; + res += n; + + if (res < (keylen - 1)) { + pr2serr("Unrecognised short option in '%s', try '--help'\n", + key); + return SG_LIB_SYNTAX_ERROR; + } + } else if (0 == strncmp(key, "--chkaddr", 9)) + ++clp->chkaddr; + else if ((0 == strncmp(key, "--dry-run", 9)) || + (0 == strncmp(key, "--dry_run", 9))) + ++clp->dry_run; + else if ((0 == strncmp(key, "--help", 6)) || + (0 == strcmp(key, "-?"))) + ++clp->help; + else if ((0 == strncmp(key, "--prefetch", 10)) || + (0 == strncmp(key, "--pre-fetch", 11))) + clp->prefetch = true; + else if (0 == strncmp(key, "--verb", 6)) { + verbose_given = true; + ++clp->verbose; /* --verbose */ + } else if (0 == strncmp(key, "--veri", 6)) + verify_given = true; + else if (0 == strncmp(key, "--vers", 6)) + version_given = true; + else { + pr2serr("Unrecognized option '%s'\n", key); + pr2serr("For more information use '--help' or '-h'\n"); + return SG_LIB_SYNTAX_ERROR; + } + } + +#ifdef DEBUG + pr2serr("In DEBUG mode, "); + if (verbose_given && version_given) { + pr2serr("but override: '-vV' given, zero verbose and continue\n"); + verbose_given = false; + version_given = false; + clp->verbose = 0; + } else if (! verbose_given) { + pr2serr("set '-vv'\n"); + clp->verbose = 2; + } else + pr2serr("keep verbose=%d\n", clp->verbose); +#else + if (verbose_given && version_given) + pr2serr("Not in DEBUG mode, so '-vV' has no special action\n"); +#endif + if (version_given) { + pr2serr("%s%s\n", my_name, version_str); + return SG_LIB_OK_FALSE; + } + if (clp->help > 0) { + usage(clp->help); + return SG_LIB_OK_FALSE; + } + if (clp->bs <= 0) { + clp->bs = DEF_BLOCK_SIZE; + pr2serr("Assume default 'bs' ((logical) block size) of %d bytes\n", + clp->bs); + } + if (verify_given) { + pr2serr("Doing verify/cmp rather than copy\n"); + clp->verify = true; + } + if ((ibs && (ibs != clp->bs)) || (obs && (obs != clp->bs))) { + pr2serr("If 'ibs' or 'obs' given must be same as 'bs'\n"); + usage(0); + return SG_LIB_SYNTAX_ERROR; + } + if ((clp->skip < 0) || (clp->seek < 0)) { + pr2serr("skip and seek cannot be negative\n"); + return SG_LIB_SYNTAX_ERROR; + } + if (clp->out_flags.append) { + if (clp->seek > 0) { + pr2serr("Can't use both append and seek switches\n"); + return SG_LIB_SYNTAX_ERROR; + } + if (verify_given) { + pr2serr("Can't use both append and verify switches\n"); + return SG_LIB_SYNTAX_ERROR; + } + } + if (clp->bpt < 1) { + pr2serr("bpt must be greater than 0\n"); + return SG_LIB_SYNTAX_ERROR; + } + if (clp->in_flags.mmap && clp->out_flags.mmap) { + pr2serr("mmap flag on both IFILE and OFILE doesn't work\n"); + return SG_LIB_SYNTAX_ERROR; + } + if (! clp->noshare) { + if (clp->in_flags.noshare || clp->out_flags.noshare) + clp->noshare = true; + } + if (clp->unshare) { + if (clp->in_flags.no_unshare || clp->out_flags.no_unshare) + clp->unshare = false; + } + if (clp->out_flags.mmap && ! clp->noshare) { + pr2serr("oflag=mmap needs either noshare=1\n"); + return SG_LIB_SYNTAX_ERROR; + } + if ((clp->in_flags.mmap || clp->out_flags.mmap) && + (clp->in_flags.same_fds || clp->out_flags.same_fds)) { + pr2serr("can't have both 'mmap' and 'same_fds' flags\n"); + return SG_LIB_SYNTAX_ERROR; + } + if ((! clp->noshare) && (clp->in_flags.dio || clp->out_flags.dio)) { + pr2serr("dio flag can only be used with noshare=1\n"); + return SG_LIB_SYNTAX_ERROR; + } + if (clp->nmrqs > 0) { + if (clp->in_flags.mrq_immed || clp->out_flags.mrq_immed) + clp->mrq_async = true; + } + /* defaulting transfer size to 128*2048 for CD/DVDs is too large + for the block layer in lk 2.6 and results in an EIO on the + SG_IO ioctl. So reduce it in that case. */ + if ((clp->bs >= 2048) && (! bpt_given)) + clp->bpt = DEF_BLOCKS_PER_2048TRANSFER; + if (clp->ofsplit >= clp->bpt) { + pr2serr("ofsplit when given must be less than BPT\n"); + return SG_LIB_SYNTAX_ERROR; + } + if ((num_threads < 1) || (num_threads > MAX_NUM_THREADS)) { + pr2serr("too few or too many threads requested\n"); + usage(1); + return SG_LIB_SYNTAX_ERROR; + } + if (clp->in_flags.swait || clp->out_flags.swait) { + if (clp->verbose) + pr2serr("the 'swait' flag is now ignored\n"); + /* remnants ... */ + if (clp->in_flags.swait && (! clp->out_flags.swait)) + clp->out_flags.swait = true; + } + clp->unit_nanosec = (do_time > 1) || !!getenv("SG3_UTILS_LINUX_NANO"); +#if 0 + if (clp->verbose) { + pr2serr("%sif=%s skip=%" PRId64 " of=%s seek=%" PRId64 " count=%" + PRId64, my_name, inf, clp->skip, outf, clp->seek, dd_count); + if (clp->nmrqs > 0) + pr2serr(" mrq=%d%s\n", clp->nmrqs, (clp->mrq_cmds ? ",C" : "")); + else + pr2serr("\n"); + } +#endif + return 0; +} + + +int +main(int argc, char * argv[]) +{ + char inf[INOUTF_SZ]; + char outf[INOUTF_SZ]; + char out2f[INOUTF_SZ]; + char outregf[INOUTF_SZ]; + int res, k, err; + int64_t in_num_sect = 0; + int64_t out_num_sect = 0; + int in_sect_sz, out_sect_sz, status, flags; + void * vp; + const char * ccp = NULL; + const char * cc2p; + struct global_collection * clp = &gcoll; + Thread_info thread_arr[MAX_NUM_THREADS] {}; + char ebuff[EBUFF_SZ]; +#if SG_LIB_ANDROID + struct sigaction actions; + + memset(&actions, 0, sizeof(actions)); + sigemptyset(&actions.sa_mask); + actions.sa_flags = 0; + actions.sa_handler = thread_exit_handler; + sigaction(SIGUSR1, &actions, NULL); + sigaction(SIGUSR2, &actions, NULL); +#endif + /* memset(clp, 0, sizeof(*clp)); */ + clp->bpt = DEF_BLOCKS_PER_TRANSFER; + clp->cmd_timeout = DEF_TIMEOUT; + clp->in_type = FT_OTHER; + /* change dd's default: if of=OFILE not given, assume /dev/null */ + clp->out_type = FT_DEV_NULL; + clp->out2_type = FT_DEV_NULL; + clp->cdbsz_in = DEF_SCSI_CDBSZ; + clp->cdbsz_out = DEF_SCSI_CDBSZ; + clp->sdt_ict = DEF_SDT_ICT_MS; + clp->sdt_crt = DEF_SDT_CRT_SEC; + clp->nmrqs = DEF_NUM_MRQS; + clp->unshare = true; + inf[0] = '\0'; + outf[0] = '\0'; + out2f[0] = '\0'; + outregf[0] = '\0'; + fetch_sg_version(); + if (sg_version >= 40045) + sg_version_ge_40045 = true; + + res = parse_cmdline_sanity(argc, argv, clp, inf, outf, out2f, outregf); + if (SG_LIB_OK_FALSE == res) + return 0; + if (res) + return res; + if (sg_version > 40000) { + if (! clp->in_flags.v3) + clp->in_flags.v4 = true; + if (! clp->out_flags.v3) + clp->out_flags.v4 = true; + } + + install_handler(SIGINT, interrupt_handler); + install_handler(SIGQUIT, interrupt_handler); + install_handler(SIGPIPE, interrupt_handler); + install_handler(SIGUSR1, siginfo_handler); + install_handler(SIGUSR2, siginfo2_handler); + + clp->infd = STDIN_FILENO; + clp->outfd = STDOUT_FILENO; + if (clp->in_flags.ff && clp->in_flags.zero) { + ccp = "<addr_as_data>"; + cc2p = "addr_as_data"; + } else if (clp->in_flags.ff) { + ccp = "<0xff bytes>"; + cc2p = "ff"; + } else if (clp->in_flags.random) { + ccp = "<random>"; + cc2p = "random"; + } else if (clp->in_flags.zero) { + ccp = "<zero bytes>"; + cc2p = "00"; + } + if (ccp) { + if (inf[0]) { + pr2serr("%siflag=%s and if=%s contradict\n", my_name, cc2p, inf); + return SG_LIB_CONTRADICT; + } + clp->in_type = FT_RANDOM_0_FF; + clp->infp = ccp; + clp->infd = -1; + } else if (inf[0] && ('-' != inf[0])) { + clp->in_type = dd_filetype(inf, clp->in_st_size); + + if (FT_ERROR == clp->in_type) { + pr2serr("%sunable to access %s\n", my_name, inf); + return SG_LIB_FILE_ERROR; + } else if (FT_ST == clp->in_type) { + pr2serr("%sunable to use scsi tape device %s\n", my_name, inf); + return SG_LIB_FILE_ERROR; + } else if (FT_CHAR == clp->in_type) { + pr2serr("%sunable to use unknown char device %s\n", my_name, inf); + return SG_LIB_FILE_ERROR; + } else if (FT_SG == clp->in_type) { + clp->infd = sg_in_open(clp, inf, NULL, NULL); + if (clp->verbose > 2) + pr2serr("using sg v%c interface on %s\n", + (clp->in_flags.v4 ? '4' : '3'), inf); + if (clp->infd < 0) + return -clp->infd; + } else { + flags = O_RDONLY; + if (clp->in_flags.direct) + flags |= O_DIRECT; + if (clp->in_flags.excl) + flags |= O_EXCL; + if (clp->in_flags.dsync) + flags |= O_SYNC; + + if ((clp->infd = open(inf, flags)) < 0) { + err = errno; + snprintf(ebuff, EBUFF_SZ, "%scould not open %s for reading", + my_name, inf); + perror(ebuff); + return sg_convert_errno(err); + } else if (clp->skip > 0) { + off64_t offset = clp->skip; + + offset *= clp->bs; /* could exceed 32 here! */ + if (lseek64(clp->infd, offset, SEEK_SET) < 0) { + err = errno; + snprintf(ebuff, EBUFF_SZ, "%scouldn't skip to required " + "position on %s", my_name, inf); + perror(ebuff); + return sg_convert_errno(err); + } + } + } + clp->infp = inf; + if ((clp->in_flags.v3 || clp->in_flags.v4_given) && + (FT_SG != clp->in_type)) { + clp->in_flags.v3 = false; + clp->in_flags.v4 = false; + pr2serr("%siflag= v3 and v4 both ignored when IFILE is not sg " + "device\n", my_name); + } + } + if (clp->verbose && (clp->in_flags.no_waitq || clp->out_flags.no_waitq)) + pr2serr("no_waitq: flag no longer does anything\n"); + if (outf[0]) + clp->ofile_given = true; + if (outf[0] && ('-' != outf[0])) { + clp->out_type = dd_filetype(outf, clp->out_st_size); + + if ((FT_SG != clp->out_type) && clp->verify) { + pr2serr("%s --verify only supported by sg OFILEs\n", my_name); + return SG_LIB_FILE_ERROR; + } else if (FT_ST == clp->out_type) { + pr2serr("%sunable to use scsi tape device %s\n", my_name, outf); + return SG_LIB_FILE_ERROR; + } else if (FT_CHAR == clp->out_type) { + pr2serr("%sunable to use unknown char device %s\n", my_name, outf); + return SG_LIB_FILE_ERROR; + } else if (FT_SG == clp->out_type) { + clp->outfd = sg_out_open(clp, outf, NULL, NULL); + if (clp->verbose > 2) + pr2serr("using sg v%c interface on %s\n", + (clp->out_flags.v4 ? '4' : '3'), outf); + if (clp->outfd < 0) + return -clp->outfd; + } else if (FT_DEV_NULL == clp->out_type) + clp->outfd = -1; /* don't bother opening */ + else { + flags = O_WRONLY; + if (! clp->out_flags.nocreat) + flags |= O_CREAT; + if (clp->out_flags.direct) + flags |= O_DIRECT; + if (clp->out_flags.excl) + flags |= O_EXCL; + if (clp->out_flags.dsync) + flags |= O_SYNC; + if (clp->out_flags.append) + flags |= O_APPEND; + + if ((clp->outfd = open(outf, flags, 0666)) < 0) { + err = errno; + snprintf(ebuff, EBUFF_SZ, "%scould not open %s for " + "writing", my_name, outf); + perror(ebuff); + return sg_convert_errno(err); + } + if (clp->seek > 0) { + off64_t offset = clp->seek; + + offset *= clp->bs; /* could exceed 32 bits here! */ + if (lseek64(clp->outfd, offset, SEEK_SET) < 0) { + err = errno; + snprintf(ebuff, EBUFF_SZ, "%scouldn't seek to required " + "position on %s", my_name, outf); + perror(ebuff); + return sg_convert_errno(err); + } + } + } + clp->outfp = outf; + if ((clp->out_flags.v3 || clp->out_flags.v4_given) && + (FT_SG != clp->out_type)) { + clp->out_flags.v3 = false; + clp->out_flags.v4 = false; + pr2serr("%soflag= v3 and v4 both ignored when OFILE is not sg " + "device\n", my_name); + } + } + + if (out2f[0]) + clp->ofile2_given = true; + if (out2f[0] && ('-' != out2f[0])) { + off_t out2_st_size; + + clp->out2_type = dd_filetype(out2f, out2_st_size); + if (FT_ST == clp->out2_type) { + pr2serr("%sunable to use scsi tape device %s\n", my_name, out2f); + return SG_LIB_FILE_ERROR; + } + else if (FT_SG == clp->out2_type) { + clp->out2fd = sg_out_open(clp, out2f, NULL, NULL); + if (clp->out2fd < 0) + return -clp->out2fd; + } + else if (FT_DEV_NULL == clp->out2_type) + clp->out2fd = -1; /* don't bother opening */ + else { + flags = O_WRONLY; + if (! clp->out_flags.nocreat) + flags |= O_CREAT; + if (clp->out_flags.direct) + flags |= O_DIRECT; + if (clp->out_flags.excl) + flags |= O_EXCL; + if (clp->out_flags.dsync) + flags |= O_SYNC; + if (clp->out_flags.append) + flags |= O_APPEND; + + if ((clp->out2fd = open(out2f, flags, 0666)) < 0) { + err = errno; + snprintf(ebuff, EBUFF_SZ, "%scould not open %s for " + "writing", my_name, out2f); + perror(ebuff); + return sg_convert_errno(err); + } + if (clp->seek > 0) { + off64_t offset = clp->seek; + + offset *= clp->bs; /* could exceed 32 bits here! */ + if (lseek64(clp->out2fd, offset, SEEK_SET) < 0) { + err = errno; + snprintf(ebuff, EBUFF_SZ, "%scouldn't seek to required " + "position on %s", my_name, out2f); + perror(ebuff); + return sg_convert_errno(err); + } + } + } + clp->out2fp = out2f; + } + if ((FT_SG == clp->in_type ) && (FT_SG == clp->out_type)) { + if (clp->nmrqs > 0) { + if (clp->is_mrq_i == clp->is_mrq_o) { + if (clp->ofsplit > 0) { + if (0 != (clp->nmrqs % 3)) { + pr2serr("When both IFILE+OFILE sg devices and OSP>0, " + "mrq=NRQS must be divisible by 3\n"); + pr2serr(" triple NRQS to avoid error\n"); + clp->nmrqs *= 3; + } + } else if (0 != (clp->nmrqs % 2)) { + pr2serr("When both IFILE+OFILE sg devices (and OSP=0), " + "mrq=NRQS must be even\n"); + pr2serr(" double NRQS to avoid error\n"); + clp->nmrqs *= 2; + } + } + if (clp->is_mrq_i && clp->is_mrq_o) + ; + else if (clp->is_mrq_i || clp->is_mrq_o) + clp->unbalanced_mrq = true; + } + if (clp->in_flags.v4_given && (! clp->out_flags.v3)) { + if (! clp->out_flags.v4_given) { + clp->out_flags.v4 = true; + if (clp->verbose) + pr2serr("Changing OFILE from v3 to v4, use oflag=v3 to " + "force v3\n"); + } + } + if (clp->out_flags.v4_given && (! clp->in_flags.v3)) { + if (! clp->in_flags.v4_given) { + clp->in_flags.v4 = true; + if (clp->verbose) + pr2serr("Changing IFILE from v3 to v4, use iflag=v3 to " + "force v3\n"); + } + } +#if 0 + if (clp->mrq_async && !(clp->noshare)) { + pr2serr("With mrq_immed also need noshare on sg-->sg copy\n"); + return SG_LIB_SYNTAX_ERROR; + } +#endif + } else if ((FT_SG == clp->in_type ) || (FT_SG == clp->out_type)) { + if (clp->nmrqs > 0) + clp->unbalanced_mrq = true; + } + if (outregf[0]) { + off_t outrf_st_size; + int ftyp = dd_filetype(outregf, outrf_st_size); + + clp->outreg_type = ftyp; + if (! ((FT_OTHER == ftyp) || (FT_ERROR == ftyp) || + (FT_DEV_NULL == ftyp))) { + pr2serr("File: %s can only be regular file or pipe (or " + "/dev/null)\n", outregf); + return SG_LIB_SYNTAX_ERROR; + } + if ((clp->outregfd = open(outregf, O_WRONLY | O_CREAT, 0666)) < 0) { + err = errno; + snprintf(ebuff, EBUFF_SZ, "could not open %s for writing", + outregf); + perror(ebuff); + return sg_convert_errno(err); + } + if (clp->verbose > 1) + pr2serr("ofreg=%s opened okay, fd=%d\n", outregf, clp->outregfd); + if (FT_ERROR == ftyp) + clp->outreg_type = FT_OTHER; /* regular file created */ + } else + clp->outregfd = -1; + + if ((STDIN_FILENO == clp->infd) && (STDOUT_FILENO == clp->outfd)) { + pr2serr("Won't default both IFILE to stdin _and_ OFILE to " + "/dev/null\n"); + pr2serr("For more information use '--help' or '-h'\n"); + return SG_LIB_SYNTAX_ERROR; + } + if (dd_count < 0) { + in_num_sect = -1; + if (FT_SG == clp->in_type) { + res = scsi_read_capacity(clp->infd, &in_num_sect, &in_sect_sz); + if (2 == res) { + pr2serr("Unit attention, media changed(in), continuing\n"); + res = scsi_read_capacity(clp->infd, &in_num_sect, + &in_sect_sz); + } + if (0 != res) { + if (res == SG_LIB_CAT_INVALID_OP) + pr2serr("read capacity not supported on %s\n", inf); + else if (res == SG_LIB_CAT_NOT_READY) + pr2serr("read capacity failed, %s not ready\n", inf); + else + pr2serr("Unable to read capacity on %s\n", inf); + return SG_LIB_FILE_ERROR; + } else if (clp->bs != in_sect_sz) { + pr2serr(">> warning: logical block size on %s confusion: " + "bs=%d, device claims=%d\n", clp->infp, clp->bs, + in_sect_sz); + return SG_LIB_FILE_ERROR; + } + } else if (FT_BLOCK == clp->in_type) { + if (0 != read_blkdev_capacity(clp->infd, &in_num_sect, + &in_sect_sz)) { + pr2serr("Unable to read block capacity on %s\n", inf); + in_num_sect = -1; + } + if (clp->bs != in_sect_sz) { + pr2serr("logical block size on %s confusion; bs=%d, from " + "device=%d\n", inf, clp->bs, in_sect_sz); + in_num_sect = -1; + } + } else if (FT_OTHER == clp->in_type) { + in_num_sect = clp->in_st_size / clp->bs; + if (clp->in_st_size % clp->bs) { + ++in_num_sect; + pr2serr("Warning: the file size of %s is not a multiple of BS " + "[%d]\n", inf, clp->bs); + } + } + if (in_num_sect > clp->skip) + in_num_sect -= clp->skip; + + out_num_sect = -1; + if (FT_SG == clp->out_type) { + res = scsi_read_capacity(clp->outfd, &out_num_sect, &out_sect_sz); + if (2 == res) { + pr2serr("Unit attention, media changed(out), continuing\n"); + res = scsi_read_capacity(clp->outfd, &out_num_sect, + &out_sect_sz); + } + if (0 != res) { + if (res == SG_LIB_CAT_INVALID_OP) + pr2serr("read capacity not supported on %s\n", outf); + else if (res == SG_LIB_CAT_NOT_READY) + pr2serr("read capacity failed, %s not ready\n", outf); + else + pr2serr("Unable to read capacity on %s\n", outf); + out_num_sect = -1; + return SG_LIB_FILE_ERROR; + } else if (clp->bs != out_sect_sz) { + pr2serr(">> warning: logical block size on %s confusion: " + "bs=%d, device claims=%d\n", clp->outfp, clp->bs, + out_sect_sz); + return SG_LIB_FILE_ERROR; + } + } else if (FT_BLOCK == clp->out_type) { + if (0 != read_blkdev_capacity(clp->outfd, &out_num_sect, + &out_sect_sz)) { + pr2serr("Unable to read block capacity on %s\n", outf); + out_num_sect = -1; + } + if (clp->bs != out_sect_sz) { + pr2serr("logical block size on %s confusion: bs=%d, from " + "device=%d\n", outf, clp->bs, out_sect_sz); + out_num_sect = -1; + } + } else if (FT_OTHER == clp->out_type) { + out_num_sect = clp->out_st_size / clp->bs; + if (clp->out_st_size % clp->bs) { + ++out_num_sect; + pr2serr("Warning: the file size of %s is not a multiple of BS " + "[%d]\n", outf, clp->bs); + } + } + if (out_num_sect > clp->seek) + out_num_sect -= clp->seek; + + if (in_num_sect > 0) { + if (out_num_sect > 0) + dd_count = (in_num_sect > out_num_sect) ? out_num_sect : + in_num_sect; + else + dd_count = in_num_sect; + } + else + dd_count = out_num_sect; + } + if (clp->verbose > 2) + pr2serr("Start of loop, count=%" PRId64 ", in_num_sect=%" PRId64 + ", out_num_sect=%" PRId64 "\n", dd_count, in_num_sect, + out_num_sect); + if (dd_count < 0) { + pr2serr("Couldn't calculate count, please give one\n"); + return SG_LIB_CAT_OTHER; + } + if (! clp->cdbsz_given) { + if ((FT_SG == clp->in_type) && (MAX_SCSI_CDBSZ != clp->cdbsz_in) && + (((dd_count + clp->skip) > UINT_MAX) || (clp->bpt > USHRT_MAX))) { + pr2serr("Note: SCSI command size increased to 16 bytes (for " + "'if')\n"); + clp->cdbsz_in = MAX_SCSI_CDBSZ; + } + if ((FT_SG == clp->out_type) && (MAX_SCSI_CDBSZ != clp->cdbsz_out) && + (((dd_count + clp->seek) > UINT_MAX) || (clp->bpt > USHRT_MAX))) { + pr2serr("Note: SCSI command size increased to 16 bytes (for " + "'of')\n"); + clp->cdbsz_out = MAX_SCSI_CDBSZ; + } + } + + // clp->in_count = dd_count; + clp->in_rem_count = dd_count; + clp->out_count = dd_count; + clp->out_rem_count = dd_count; + clp->out_blk = clp->seek; + status = pthread_mutex_init(&clp->in_mutex, NULL); + if (0 != status) err_exit(status, "init in_mutex"); + status = pthread_mutex_init(&clp->out_mutex, NULL); + if (0 != status) err_exit(status, "init out_mutex"); + status = pthread_mutex_init(&clp->out2_mutex, NULL); + if (0 != status) err_exit(status, "init out2_mutex"); + status = pthread_cond_init(&clp->out_sync_cv, NULL); + if (0 != status) err_exit(status, "init out_sync_cv"); + + if (clp->dry_run > 0) { + pr2serr("Due to --dry-run option, bypass copy/read\n"); + goto fini; + } + if (! clp->ofile_given) + pr2serr("of=OFILE not given so only read from IFILE, to output to " + "stdout use 'of=-'\n"); + + sigemptyset(&signal_set); + sigaddset(&signal_set, SIGINT); + sigaddset(&signal_set, SIGUSR2); + status = pthread_sigmask(SIG_BLOCK, &signal_set, &orig_signal_set); + if (0 != status) err_exit(status, "pthread_sigmask"); + status = pthread_create(&sig_listen_thread_id, NULL, + sig_listen_thread, (void *)clp); + if (0 != status) err_exit(status, "pthread_create, sig..."); + + if (do_time) { + start_tm.tv_sec = 0; + start_tm.tv_usec = 0; + gettimeofday(&start_tm, NULL); + } + +/* vvvvvvvvvvv Start worker threads vvvvvvvvvvvvvvvvvvvvvvvv */ + if ((clp->out_rem_count.load() > 0) && (num_threads > 0)) { + Thread_info *tip = thread_arr + 0; + + tip->gcp = clp; + tip->id = 0; + /* Run 1 work thread to shake down infant retryable stuff */ + status = pthread_mutex_lock(&clp->out_mutex); + if (0 != status) err_exit(status, "lock out_mutex"); + status = pthread_create(&tip->a_pthr, NULL, read_write_thread, + (void *)tip); + if (0 != status) err_exit(status, "pthread_create"); + + /* wait for any broadcast */ + pthread_cleanup_push(cleanup_out, (void *)clp); + status = pthread_cond_wait(&clp->out_sync_cv, &clp->out_mutex); + if (0 != status) err_exit(status, "cond out_sync_cv"); + pthread_cleanup_pop(0); + status = pthread_mutex_unlock(&clp->out_mutex); + if (0 != status) err_exit(status, "unlock out_mutex"); + + /* now start the rest of the threads */ + for (k = 1; k < num_threads; ++k) { + tip = thread_arr + k; + tip->gcp = clp; + tip->id = k; + status = pthread_create(&tip->a_pthr, NULL, read_write_thread, + (void *)tip); + if (0 != status) err_exit(status, "pthread_create"); + } + + /* now wait for worker threads to finish */ + for (k = 0; k < num_threads; ++k) { + tip = thread_arr + k; + status = pthread_join(tip->a_pthr, &vp); + if (0 != status) err_exit(status, "pthread_join"); + if (clp->verbose > 2) + pr2serr_lk("%d <-- Worker thread terminated, vp=%s\n", k, + ((vp == clp) ? "clp" : "NULL (or !clp)")); + } + } /* started worker threads and here after they have all exited */ + + if (do_time && (start_tm.tv_sec || start_tm.tv_usec)) + calc_duration_throughput(0); + + shutting_down = true; + status = pthread_join(sig_listen_thread_id, &vp); + if (0 != status) err_exit(status, "pthread_join"); +#if 0 + /* pthread_cancel() has issues and is not supported in Android */ + status = pthread_kill(sig_listen_thread_id, SIGUSR2); + if (0 != status) err_exit(status, "pthread_kill"); + std::this_thread::yield(); // not enough it seems + { /* allow time for SIGUSR2 signal to get through */ + struct timespec tspec = {0, 400000}; /* 400 usecs */ + + nanosleep(&tspec, NULL); + } +#endif + + if (do_sync) { + if (FT_SG == clp->out_type) { + pr2serr_lk(">> Synchronizing cache on %s\n", outf); + res = sg_ll_sync_cache_10(clp->outfd, 0, 0, 0, 0, 0, false, 0); + if (SG_LIB_CAT_UNIT_ATTENTION == res) { + pr2serr_lk("Unit attention(out), continuing\n"); + res = sg_ll_sync_cache_10(clp->outfd, 0, 0, 0, 0, 0, false, + 0); + } + if (0 != res) + pr2serr_lk("Unable to synchronize cache\n"); + } + if (FT_SG == clp->out2_type) { + pr2serr_lk(">> Synchronizing cache on %s\n", out2f); + res = sg_ll_sync_cache_10(clp->out2fd, 0, 0, 0, 0, 0, false, 0); + if (SG_LIB_CAT_UNIT_ATTENTION == res) { + pr2serr_lk("Unit attention(out2), continuing\n"); + res = sg_ll_sync_cache_10(clp->out2fd, 0, 0, 0, 0, 0, false, + 0); + } + if (0 != res) + pr2serr_lk("Unable to synchronize cache (of2)\n"); + } + } + +fini: + if ((STDIN_FILENO != clp->infd) && (clp->infd >= 0)) + close(clp->infd); + if ((STDOUT_FILENO != clp->outfd) && (FT_DEV_NULL != clp->out_type) && + (clp->outfd >= 0)) + close(clp->outfd); + if ((clp->out2fd >= 0) && (STDOUT_FILENO != clp->out2fd) && + (FT_DEV_NULL != clp->out2_type)) + close(clp->out2fd); + if ((clp->outregfd >= 0) && (STDOUT_FILENO != clp->outregfd) && + (FT_DEV_NULL != clp->outreg_type)) + close(clp->outregfd); + res = exit_status; + if ((0 != clp->out_count.load()) && (0 == clp->dry_run)) { + pr2serr(">>>> Some error occurred, remaining blocks=%" PRId64 "\n", + clp->out_count.load()); + if (0 == res) + res = SG_LIB_CAT_OTHER; + } + print_stats(""); + if (clp->dio_incomplete_count.load()) { + int fd; + char c; + + pr2serr(">> Direct IO requested but incomplete %d times\n", + clp->dio_incomplete_count.load()); + if ((fd = open(sg_allow_dio, O_RDONLY)) >= 0) { + if (1 == read(fd, &c, 1)) { + if ('0' == c) + pr2serr(">>> %s set to '0' but should be set to '1' for " + "direct IO\n", sg_allow_dio); + } + close(fd); + } + } + if (clp->sum_of_resids.load()) + pr2serr(">> Non-zero sum of residual counts=%d\n", + clp->sum_of_resids.load()); + if (clp->verbose && (num_start_eagain > 0)) + pr2serr("Number of start EAGAINs: %d\n", num_start_eagain.load()); + if (clp->verbose && (num_fin_eagain > 0)) + pr2serr("Number of finish EAGAINs: %d\n", num_fin_eagain.load()); + if (clp->verbose && (num_ebusy > 0)) + pr2serr("Number of EBUSYs: %d\n", num_ebusy.load()); + if (clp->verbose && clp->aen_given && (num_abort_req > 0)) { + pr2serr("Number of Aborts: %d\n", num_abort_req.load()); + pr2serr("Number of successful Aborts: %d\n", + num_abort_req_success.load()); + } + if (clp->verbose && clp->m_aen_given && (num_mrq_abort_req > 0)) { + pr2serr("Number of MRQ Aborts: %d\n", num_mrq_abort_req.load()); + pr2serr("Number of successful MRQ Aborts: %d\n", + num_mrq_abort_req_success.load()); + } + if (clp->verbose && (num_miscompare > 0)) + pr2serr("Number of miscompare%s: %d\n", + (num_miscompare > 1) ? "s" : "", num_miscompare.load()); + if (clp->verbose > 1) { + if (clp->verbose > 3) + pr2serr("Final pack_id=%d, mrq_id=%d\n", mono_pack_id.load(), + mono_mrq_id.load()); + pr2serr("Number of SG_GET_NUM_WAITING calls=%ld\n", + num_waiting_calls.load()); + } + if (clp->verify && (SG_LIB_CAT_MISCOMPARE == res)) + pr2serr("Verify/compare failed due to miscompare\n"); + return (res >= 0) ? res : SG_LIB_CAT_OTHER; +} |