From ddc4b6d7953970648dddaa03c7a28af13c5efb4e Mon Sep 17 00:00:00 2001 From: Douglas Gilbert Date: Thu, 10 Jul 2014 16:59:45 +0000 Subject: sg_format: make '-FFF' bypass mode sense/select, add --mode=MP; trim trailing spaces in dStrHex() and friends; add examples/sg_tst_async git-svn-id: https://svn.bingwo.ca/repos/sg3_utils/trunk@591 6180dd3e-e324-4e3e-922d-17de1ae2f315 --- examples/README | 2 +- examples/sg_tst_async.cpp | 817 +++++++++++++++++++++++----------------------- examples/sg_tst_excl.cpp | 4 +- examples/sgq_dd.c | 2 +- 4 files changed, 415 insertions(+), 410 deletions(-) (limited to 'examples') diff --git a/examples/README b/examples/README index b26d8da1..0a17fb64 100644 --- a/examples/README +++ b/examples/README @@ -16,4 +16,4 @@ in C++11. They can be built by calling 'make -f Makefile.cplus'. A gcc/g++ compiler of 4.7.3 vintage or later will be required. Douglas Gilbert -21st October 2013 +10th July 2014 diff --git a/examples/sg_tst_async.cpp b/examples/sg_tst_async.cpp index ee834c51..c69446d6 100644 --- a/examples/sg_tst_async.cpp +++ b/examples/sg_tst_async.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013 Douglas Gilbert. + * Copyright (c) 2014 Douglas Gilbert. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -28,115 +28,157 @@ #include #include +#include +#include #include #include #include #include +#include #include #include #include #include #include +#include #include #include +#include +#define __STDC_FORMAT_MACROS 1 +#include #include #include #include #include "sg_lib.h" #include "sg_io_linux.h" -static const char * version_str = "1.00 20140614"; +static const char * version_str = "1.00 20140710"; static const char * util_name = "sg_tst_async"; /* This is a test program for checking the async usage of the Linux sg - * driver. - * multiple threads and can be run as multiple processes and attempts - * to "break" O_EXCL. The strategy is to open a device O_EXCL|O_NONBLOCK - * and do a double increment on a LB then close it. Prior to the first - * increment, the value is checked for even or odd. Assuming the count - * starts as an even (typically 0) then it should remain even. Odd instances - * are counted and reported at the end of the program, after all threads - * have completed. + * driver. Each thread opens 1 file descriptor to the sg device and then + * starts up to 16 commands while checking with the poll command for + * the completion of those commands. Each command has a unique "pack_id" + * which is a sequence starting at 1. Either TEST UNIT UNIT, READ(16) + * or WRITE(16) commands are issued. * * This is C++ code with some things from C++11 (e.g. threads) and was * only just able to compile (when some things were reverted) with gcc/g++ * version 4.7.3 found in Ubuntu 13.04 . C++11 "feature complete" support - * was not available until g++ version 4.8.1 and that is only currently - * found in Fedora 19 . + * was not available until g++ version 4.8.1 . It should build okay on + * recent distributions. * * The build uses various object files from the /lib directory * which is assumed to be a sibling of this examples directory. Those * object files in the lib directory can be built with: * cd ; ./configure ; cd lib; make - * Then to build sg_tst_excl concatenate the next 3 lines: + * Then to build sg_tst_async concatenate the next 3 lines: * g++ -Wall -std=c++11 -pthread -I ../include ../lib/sg_lib.o - * ../lib/sg_lib_data.o ../lib/sg_io_linux.o -o sg_tst_excl - * sg_tst_excl.cpp + * ../lib/sg_lib_data.o ../lib/sg_io_linux.o -o sg_tst_async + * sg_tst_async.cpp + * or use the C++ Makefile in that directory: + * make -f Makefile.cplus sg_tst_async * - * Currently this utility is Linux only and assumes the SG_IO v3 interface - * which is supported by sg and block devices (but not bsg devices which - * require the SG_IO v4 interface). This restriction is relaxed in the - * sg_tst_excl2 variant of this utility. + * Currently this utility is Linux only and uses the sg driver. The bsg + * driver is known to be broken (it doesn't match responses to the + * correct file descriptor that requested them) so this utility won't + * be extended to bsg until that if fixed. * - * BEWARE: this utility modifies a logical block (default LBA 1000) on the - * given device. + * BEWARE: this utility will modify a logical block (default LBA 1000) on the + * given device when the '-W' option is given. * */ using namespace std; using namespace std::chrono; -#define DEF_NUM_PER_THREAD 200 +#define DEF_NUM_PER_THREAD 1000 #define DEF_NUM_THREADS 4 -#define DEF_WAIT_MS 0 /* 0: yield; -1: don't wait; -2: sleep(0) */ +#define DEF_WAIT_MS 10 /* 0: yield; -1: don't wait; -2: sleep(0) */ +#define DEF_TIMEOUT_MS 20000 /* 20 seconds */ +#define DEF_LB_SZ 512 +#define DEF_BLOCKING 0 +#define DEF_DIRECT 0 +#define DEF_NO_XFER 0 + +#define Q_PER_FD 16 + +#ifndef SG_FLAG_Q_AT_TAIL +#define SG_FLAG_Q_AT_TAIL 0x10 +#endif +#ifndef SG_FLAG_Q_AT_HEAD +#define SG_FLAG_Q_AT_HEAD 0x20 +#endif #define DEF_LBA 1000 #define EBUFF_SZ 256 -static mutex odd_count_mutex; static mutex console_mutex; -static unsigned int odd_count; -static unsigned int ebusy_count; -static unsigned int eagain_count; +static atomic async_starts(0); +static atomic async_finishes(0); +static atomic ebusy_count(0); +static atomic eagain_count(0); +static atomic uniq_pack_id(1); + +static int page_size = 4096; /* rough guess, will ask sysconf() */ + +enum command2execute {SCSI_TUR, SCSI_READ16, SCSI_WRITE16}; +enum blkQDiscipline {BQ_DEFAULT, BQ_AT_HEAD, BQ_AT_TAIL}; + +struct opts_t { + const char * dev_name; + bool direct; + int num_per_thread; + bool block; + uint64_t lba; + int lb_sz; + bool no_xfer; + int verbose; + int wait_ms; + command2execute c2e; + blkQDiscipline bqd; +}; static void usage(void) { - printf("Usage: %s [-b] [-f] [-h] [-l ] [-n ] " - "[-t ]\n" - " [-V] [-w ] [-x] [-xx] " - "\n", util_name); + printf("Usage: %s [-d] [-f] [-h] [-l ] [-n ] [-N]\n" + " [-q 0|1] [-R] [-s ] [-t ] " + "[-T]\n" + " [-v] [-V] [-w ] [-W] " + "\n", + util_name); printf(" where\n"); - printf(" -b block on open (def: O_NONBLOCK)\n"); - printf(" -f force: any SCSI disk (def: only " - "scsi_debug)\n"); - printf(" WARNING: written to\n"); + printf(" -d do direct_io (def: indirect)\n"); + printf(" -f force: any sg device (def: only scsi_debug " + "owned)\n"); + printf(" WARNING: written to if '-W' given\n"); printf(" -h print this usage message then exit\n"); - printf(" -l logical block to increment (def: %u)\n", + printf(" -l logical block to access (def: %u)\n", DEF_LBA); - printf(" -n number of loops per thread " + printf(" -n number of commands per thread " "(def: %d)\n", DEF_NUM_PER_THREAD); + printf(" -N no data xfer (def: xfer on READ and " + "WRITE)\n"); + printf(" -q 0|1 0: blk q_at_head; 1: q_at_tail\n"); + printf(" -s logical block size (def: 512)\n"); + printf(" -R do READs (def: TUR)\n"); printf(" -t number of threads (def: %d)\n", DEF_NUM_THREADS); + printf(" -T do TEST UNIT READYs (default is TURs)\n"); + printf(" -v increase verbosity\n"); printf(" -V print version number then exit\n"); - printf(" -w >0: sleep_for(); =0: " - "yield(); -1: no\n" - " wait; -2: sleep(0) (def: %d)\n", - DEF_WAIT_MS); - printf(" -x don't use O_EXCL on first thread " - "(def: use\n" - " O_EXCL on all threads)\n" - " -xx don't use O_EXCL on any thread\n\n"); - printf("Test O_EXCL open flag with Linux sg driver. Each open/close " - "cycle with the\nO_EXCL flag does a double increment on " - "lba (using its first 4 bytes).\nEach increment uses a READ_16, " - "READ_16, increment, WRITE_16 cycle. The two\nREAD_16s are " - "launched asynchronously. Note that '-xx' will run test\n" - "without any O_EXCL flags.\n"); + printf(" -w >0: poll(); =0: poll(0); (def: " + "%d)\n", DEF_WAIT_MS); + printf(" -W do WRITEs (def: TUR)\n\n"); + printf("Multiple threads do READ(16), WRITE(16) or TEST UNIT READY " + "(TUR) SCSI\ncommands. Each thread has its own file descriptor " + "and queues up to\n16 commands. One block is transferred by " + "each READ and WRITE; zeros\nare written.\n"); } @@ -146,9 +188,10 @@ usage(void) #define WRITE16_REPLY_LEN 512 #define WRITE16_CMD_LEN 16 +/* Returns 0 if command injected okay, else -1 */ static int -start_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int pack_id, unsigned int lba, - unsigned char * lbp, int xfer_bytes) +start_sg3_cmd(int sg_fd, command2execute cmd2exe, int pack_id, uint64_t lba, + unsigned char * lbp, int xfer_bytes, int flags) { struct sg_io_hdr pt; unsigned char turCmdBlk[TUR_CMD_LEN] = {0, 0, 0, 0, 0, 0}; @@ -160,15 +203,21 @@ start_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int pack_id, unsigned int lba, const char * np; memset(&pt, 0, sizeof(pt)); - switch (tur0_rd1_wr2) { - case 0: + switch (cmd2exe) { + case SCSI_TUR: np = "TEST UNIT READY"; pt.cmdp = turCmdBlk; pt.cmd_len = sizeof(turCmdBlk); pt.dxfer_direction = SG_DXFER_NONE; break; - case 1: + case SCSI_READ16: np = "READ(16)"; + if (lba > 0xffffffff) { + r16CmdBlk[2] = (lba >> 56) & 0xff; + r16CmdBlk[3] = (lba >> 48) & 0xff; + r16CmdBlk[4] = (lba >> 40) & 0xff; + r16CmdBlk[5] = (lba >> 32) & 0xff; + } r16CmdBlk[6] = (lba >> 24) & 0xff; r16CmdBlk[7] = (lba >> 16) & 0xff; r16CmdBlk[8] = (lba >> 8) & 0xff; @@ -179,8 +228,14 @@ start_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int pack_id, unsigned int lba, pt.dxferp = lbp; pt.dxfer_len = xfer_bytes; break; - case 2: + case SCSI_WRITE16: np = "WRITE(16)"; + if (lba > 0xffffffff) { + w16CmdBlk[2] = (lba >> 56) & 0xff; + w16CmdBlk[3] = (lba >> 48) & 0xff; + w16CmdBlk[4] = (lba >> 40) & 0xff; + w16CmdBlk[5] = (lba >> 32) & 0xff; + } w16CmdBlk[6] = (lba >> 24) & 0xff; w16CmdBlk[7] = (lba >> 16) & 0xff; w16CmdBlk[8] = (lba >> 8) & 0xff; @@ -191,87 +246,50 @@ start_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int pack_id, unsigned int lba, pt.dxferp = lbp; pt.dxfer_len = xfer_bytes; break; - default: - console_mutex.lock(); - cerr << __func__ << ": unknown tur0_rd1_wr2=" << tur0_rd1_wr2 << endl; - console_mutex.unlock(); - return -99; } pt.interface_id = 'S'; pt.mx_sb_len = sizeof(sense_buffer); pt.sbp = sense_buffer; /* ignored .... */ - pt.timeout = 20000; /* 20000 millisecs == 20 seconds */ + pt.timeout = DEF_TIMEOUT_MS; pt.pack_id = pack_id; + pt.flags = flags; - // queue up two READ_16s to same LBA if (write(sg_fd, &pt, sizeof(pt)) < 0) { console_mutex.lock(); cerr << __func__ << ": " << np << " pack_id=" << pack_id; perror(" write(sg)"); console_mutex.unlock(); - close(sg_fd); return -1; } return 0; } static int -finish_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int & pack_id, unsigned int lba, - unsigned char * lbp, int xfer_bytes, int wait_ms, int & eagains) +finish_sg3_cmd(int sg_fd, command2execute cmd2exe, int & pack_id, int wait_ms, + unsigned int & eagains) { int ok, res; struct sg_io_hdr pt; - unsigned char turCmdBlk[TUR_CMD_LEN] = {0, 0, 0, 0, 0, 0}; - unsigned char r16CmdBlk[READ16_CMD_LEN] = - {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}; - unsigned char w16CmdBlk[WRITE16_CMD_LEN] = - {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}; unsigned char sense_buffer[64]; - const char * np; + const char * np = NULL; memset(&pt, 0, sizeof(pt)); - switch (tur0_rd1_wr2) { - case 0: + switch (cmd2exe) { + case SCSI_TUR: np = "TEST UNIT READY"; - pt.cmdp = turCmdBlk; - pt.cmd_len = sizeof(turCmdBlk); - pt.dxfer_direction = SG_DXFER_NONE; break; - case 1: + case SCSI_READ16: np = "READ(16)"; - r16CmdBlk[6] = (lba >> 24) & 0xff; - r16CmdBlk[7] = (lba >> 16) & 0xff; - r16CmdBlk[8] = (lba >> 8) & 0xff; - r16CmdBlk[9] = lba & 0xff; - pt.cmdp = r16CmdBlk; - pt.cmd_len = sizeof(r16CmdBlk); - pt.dxfer_direction = SG_DXFER_FROM_DEV; - pt.dxferp = lbp; - pt.dxfer_len = xfer_bytes; break; - case 2: + case SCSI_WRITE16: np = "WRITE(16)"; - w16CmdBlk[6] = (lba >> 24) & 0xff; - w16CmdBlk[7] = (lba >> 16) & 0xff; - w16CmdBlk[8] = (lba >> 8) & 0xff; - w16CmdBlk[9] = lba & 0xff; - pt.cmdp = w16CmdBlk; - pt.cmd_len = sizeof(w16CmdBlk); - pt.dxfer_direction = SG_DXFER_TO_DEV; - pt.dxferp = lbp; - pt.dxfer_len = xfer_bytes; break; - default: - console_mutex.lock(); - cerr << __func__ << ": unknown tur0_rd1_wr2=" << tur0_rd1_wr2 << endl; - console_mutex.unlock(); - return -99; } pt.interface_id = 'S'; pt.mx_sb_len = sizeof(sense_buffer); pt.sbp = sense_buffer; - pt.timeout = 20000; /* 20000 millisecs == 20 seconds */ - pt.pack_id = pack_id; + pt.timeout = DEF_TIMEOUT_MS; + pt.pack_id = 0; while (((res = read(sg_fd, &pt, sizeof(pt))) < 0) && (EAGAIN == errno)) { @@ -287,10 +305,10 @@ finish_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int & pack_id, unsigned int lba, console_mutex.lock(); perror("do_rd_inc_wr_twice: read(sg, READ_16)"); console_mutex.unlock(); - close(sg_fd); return -1; } /* now for the error processing */ + pack_id = pt.pack_id; ok = 0; switch (sg_err_category3(&pt)) { case SG_LIB_CAT_CLEAN: @@ -299,7 +317,7 @@ finish_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int & pack_id, unsigned int lba, case SG_LIB_CAT_RECOVERED: console_mutex.lock(); fprintf(stderr, "%s: Recovered error on %s, continuing\n", - __func__, np); + __func__, np); console_mutex.unlock(); ok = 1; break; @@ -312,230 +330,195 @@ finish_sg3_cmd(int sg_fd, int tur0_rd1_wr2, int & pack_id, unsigned int lba, return ok ? 0 : -1; } -/* Opens dev_name and spins if busy (i.e. gets EBUSY), sleeping for - * wait_ms milliseconds if wait_ms is positive. - * Reads lba (twice) and treats the first 4 bytes as an int (SCSI endian), - * increments it and writes it back. Repeats so that happens twice. Then - * closes dev_name. If an error occurs returns -1 else returns 0 if - * first int read from lba is even otherwise returns 1. */ -static int -do_rd_inc_wr_twice(const char * dev_name, unsigned int lba, int block, - int excl, int wait_ms, int id, unsigned int & ebusy, - unsigned int & eagains) +/* Should have page alignment if direct_io chosen */ +static unsigned char * +get_aligned_heap(int bytes_at_least) { - int k, sg_fd, ok, res; - int odd = 0; - unsigned int u = 0; - struct sg_io_hdr pt, pt2; - unsigned char r16CmdBlk [READ16_CMD_LEN] = - {0x88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}; - unsigned char w16CmdBlk [WRITE16_CMD_LEN] = - {0x8a, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0}; - unsigned char sense_buffer[64]; - unsigned char lb[READ16_REPLY_LEN]; - char ebuff[EBUFF_SZ]; + int n; + void * wp; + + if (bytes_at_least < page_size) + n = page_size; + else + n = bytes_at_least; +#if 1 + int err = posix_memalign(&wp, page_size, n); + if (err) { + console_mutex.lock(); + fprintf(stderr, "posix_memalign: error [%d] out of memory?\n", err); + console_mutex.unlock(); + return NULL; + } + memset(wp, 0, n); + return (unsigned char *)wp; +#else + if (n == page_size) { + wp = calloc(page_size, 1); + memset(wp, 0, n); + return (unsigned char *)wp; + } else { + console_mutex.lock(); + fprintf(stderr, "get_aligned_heap: too fiddly to align, choose " + "smaller lb_sz\n"); + console_mutex.unlock(); + return NULL; + } +#endif +} + +static void +work_thread(int id, struct opts_t * op) +{ + int thr_async_starts = 0; + int thr_async_finishes = 0; + unsigned int thr_eagain_count = 0; + int k, res, sg_fd, num_outstanding, do_inc, num, pack_id, sg_flags; int open_flags = O_RDWR; + char ebuff[EBUFF_SZ]; + unsigned char * lbp; + const char * err = NULL; + struct pollfd pfd; + list free_lst; + map pi_map; - r16CmdBlk[6] = w16CmdBlk[6] = (lba >> 24) & 0xff; - r16CmdBlk[7] = w16CmdBlk[7] = (lba >> 16) & 0xff; - r16CmdBlk[8] = w16CmdBlk[8] = (lba >> 8) & 0xff; - r16CmdBlk[9] = w16CmdBlk[9] = lba & 0xff; - if (! block) + if (op->verbose) { + console_mutex.lock(); + cerr << "Enter work_thread id=" << id << endl; + console_mutex.unlock(); + } + if (! op->block) open_flags |= O_NONBLOCK; - if (excl) - open_flags |= O_EXCL; - while (((sg_fd = open(dev_name, open_flags)) < 0) && - (EBUSY == errno)) { - ++ebusy; - if (wait_ms > 0) - this_thread::sleep_for(milliseconds{wait_ms}); - else if (0 == wait_ms) - this_thread::yield(); - else if (-2 == wait_ms) - sleep(0); // process yield ?? - } + sg_fd = open(op->dev_name, open_flags); if (sg_fd < 0) { - snprintf(ebuff, EBUFF_SZ, - "do_rd_inc_wr_twice: error opening file: %s", dev_name); + snprintf(ebuff, EBUFF_SZ, "%s: id=%d, error opening file: %s", + __func__, id, op->dev_name); + console_mutex.lock(); perror(ebuff); - return -1; + console_mutex.unlock(); + return; + } + pfd.fd = sg_fd; + pfd.events = POLLIN; + sg_flags = 0; + if (BQ_AT_TAIL == op->bqd) + sg_flags |= SG_FLAG_Q_AT_TAIL; + else if (BQ_AT_HEAD == op->bqd) + sg_flags |= SG_FLAG_Q_AT_HEAD; + if (op->direct) + sg_flags |= SG_FLAG_DIRECT_IO; + if (op->no_xfer) + sg_flags |= SG_FLAG_NO_DXFER; + if (op->verbose > 1) { + console_mutex.lock(); + fprintf(stderr, "sg_flags=0x%x, %s cmd\n", sg_flags, + ((SCSI_TUR != op->c2e) ? "TUR": "IO")); + console_mutex.unlock(); } - for (k = 0; k < 2; ++k) { - /* Prepare READ_16 command */ - memset(&pt, 0, sizeof(pt)); - pt.interface_id = 'S'; - pt.cmd_len = sizeof(r16CmdBlk); - pt.mx_sb_len = sizeof(sense_buffer); - pt.dxfer_direction = SG_DXFER_FROM_DEV; - pt.dxfer_len = READ16_REPLY_LEN; - pt.dxferp = lb; - pt.cmdp = r16CmdBlk; - pt.sbp = sense_buffer; - pt.timeout = 20000; /* 20000 millisecs == 20 seconds */ - pt.pack_id = id; - - // queue up two READ_16s to same LBA - if (write(sg_fd, &pt, sizeof(pt)) < 0) { - console_mutex.lock(); - perror("do_rd_inc_wr_twice: write(sg, READ_16)"); - console_mutex.unlock(); - close(sg_fd); - return -1; - } - pt2 = pt; - if (write(sg_fd, &pt2, sizeof(pt2)) < 0) { - console_mutex.lock(); - perror("do_rd_inc_wr_twice: write(sg, READ_16) 2"); - console_mutex.unlock(); - close(sg_fd); - return -1; - } - - while (((res = read(sg_fd, &pt, sizeof(pt))) < 0) && - (EAGAIN == errno)) { - ++eagains; - if (wait_ms > 0) - this_thread::sleep_for(milliseconds{wait_ms}); - else if (0 == wait_ms) - this_thread::yield(); - else if (-2 == wait_ms) - sleep(0); // process yield ?? - } - if (res < 0) { - console_mutex.lock(); - perror("do_rd_inc_wr_twice: read(sg, READ_16)"); - console_mutex.unlock(); - close(sg_fd); - return -1; - } - /* now for the error processing */ - ok = 0; - switch (sg_err_category3(&pt)) { - case SG_LIB_CAT_CLEAN: - ok = 1; - break; - case SG_LIB_CAT_RECOVERED: - console_mutex.lock(); - fprintf(stderr, "Recovered error on READ_16, continuing\n"); - console_mutex.unlock(); - ok = 1; - break; - default: /* won't bother decoding other categories */ - console_mutex.lock(); - sg_chk_n_print3("READ_16 command error", &pt, 1); - console_mutex.unlock(); - break; - } - if (ok) { - while (((res = read(sg_fd, &pt2, sizeof(pt2))) < 0) && - (EAGAIN == errno)) { - ++eagains; - if (wait_ms > 0) - this_thread::sleep_for(milliseconds{wait_ms}); - else if (0 == wait_ms) - this_thread::yield(); - else if (-2 == wait_ms) - sleep(0); // process yield ?? + num = op->num_per_thread; + for (k = 0, num_outstanding = 0; (k < num) || num_outstanding; + k = do_inc ? k + 1 : k) { + do_inc = 0; + if ((num_outstanding < Q_PER_FD) && (k < num)) { + do_inc = 1; + pack_id = uniq_pack_id.fetch_add(1); + if (SCSI_TUR != op->c2e) { + if (free_lst.empty()) { + lbp = get_aligned_heap(op->lb_sz); + if (NULL == lbp) { + err = "out of memory"; + break; + } + } else { + lbp = free_lst.back(); + free_lst.pop_back(); + } + } else + lbp = NULL; + if (start_sg3_cmd(sg_fd, op->c2e, pack_id, op->lba, lbp, + op->lb_sz, sg_flags)) { + err = "start_sg3_cmd() failed"; + break; } + ++thr_async_starts; + ++num_outstanding; + pi_map[pack_id] = lbp; + /* check if any responses, don't wait */ + res = poll(&pfd, 1, 0); if (res < 0) { - console_mutex.lock(); - perror("do_rd_inc_wr_twice: read(sg, READ_16) 2"); - console_mutex.unlock(); - close(sg_fd); - return -1; + err = "poll(0) failed"; + break; } - pt = pt2; - /* now for the error processing */ - ok = 0; - switch (sg_err_category3(&pt)) { - case SG_LIB_CAT_CLEAN: - ok = 1; + } else { + /* check if any responses, wait as requested */ + res = poll(&pfd, 1, ((op->wait_ms > 0) ? op->wait_ms : 0)); + if (res < 0) { + err = "poll(wait_ms) failed"; break; - case SG_LIB_CAT_RECOVERED: - console_mutex.lock(); - fprintf(stderr, "Recovered error on READ_16, continuing 2\n"); - console_mutex.unlock(); - ok = 1; + } + } + if (0 == res) + continue; + while (res-- > 0) { + if (finish_sg3_cmd(sg_fd, op->c2e, pack_id, op->wait_ms, + thr_eagain_count)) { + err = "finish_sg3_cmd() failed"; break; - default: /* won't bother decoding other categories */ - console_mutex.lock(); - sg_chk_n_print3("READ_16 command error 2", &pt, 1); - console_mutex.unlock(); + } + ++thr_async_finishes; + --num_outstanding; + auto p = pi_map.find(pack_id); + + if (p == pi_map.end()) { + snprintf(ebuff, sizeof(ebuff), "pack_id=%d from " + "finish_sg3_cmd() not found\n", pack_id); + err = ebuff; break; + } else { + lbp = p->second; + pi_map.erase(p); + if (lbp) + free_lst.push_front(lbp); } } - if (! ok) { - close(sg_fd); - return -1; - } - - u = (lb[0] << 24) + (lb[1] << 16) + (lb[2] << 8) + lb[3]; - if (0 == k) - odd = (1 == (u % 2)); - ++u; - lb[0] = (u >> 24) & 0xff; - lb[1] = (u >> 16) & 0xff; - lb[2] = (u >> 8) & 0xff; - lb[3] = u & 0xff; - - if (wait_ms > 0) /* allow daylight for bad things ... */ - this_thread::sleep_for(milliseconds{wait_ms}); - else if (0 == wait_ms) - this_thread::yield(); - else if (-2 == wait_ms) - sleep(0); // process yield ?? - - /* Prepare WRITE_16 command */ - memset(&pt, 0, sizeof(pt)); - pt.interface_id = 'S'; - pt.cmd_len = sizeof(w16CmdBlk); - pt.mx_sb_len = sizeof(sense_buffer); - pt.dxfer_direction = SG_DXFER_TO_DEV; - pt.dxfer_len = WRITE16_REPLY_LEN; - pt.dxferp = lb; - pt.cmdp = w16CmdBlk; - pt.sbp = sense_buffer; - pt.timeout = 20000; /* 20000 millisecs == 20 seconds */ - pt.pack_id = id; - - if (ioctl(sg_fd, SG_IO, &pt) < 0) { - console_mutex.lock(); - perror("do_rd_inc_wr_twice: WRITE_16 SG_IO ioctl error"); - console_mutex.unlock(); - close(sg_fd); - return -1; - } - /* now for the error processing */ - ok = 0; - switch (sg_err_category3(&pt)) { - case SG_LIB_CAT_CLEAN: - ok = 1; - break; - case SG_LIB_CAT_RECOVERED: - console_mutex.lock(); - fprintf(stderr, "Recovered error on WRITE_16, continuing\n"); - console_mutex.unlock(); - ok = 1; - break; - default: /* won't bother decoding other categories */ - console_mutex.lock(); - sg_chk_n_print3("WRITE_16 command error", &pt, 1); - console_mutex.unlock(); - break; - } - if (! ok) { - close(sg_fd); - return -1; - } } close(sg_fd); - return odd; + if (err || (k < num) || (op->verbose > 0)) { + console_mutex.lock(); + if (k < num) { + cerr << "thread id=" << id << " FAILed at iteration: " << k; + if (err) + cerr << " Reason: " << err << endl; + else + cerr << endl; + } else { + if (err) + cerr << "thread id=" << id << " FAILed on last, " << + "Reason: " << err << endl; + else + cerr << "thread id=" << id << " normal exit" << '\n'; + } + console_mutex.unlock(); + } + k = pi_map.size(); + if (k > 0) { + console_mutex.lock(); + cerr << "thread id=" << id << " Still " << k << " elements " << + "in pack_id map on exit" << endl; + console_mutex.unlock(); + } + while (! free_lst.empty()) { + lbp = free_lst.back(); + free_lst.pop_back(); + if (lbp) + free(lbp); + } + async_starts += thr_async_starts; + async_finishes += thr_async_finishes; + eagain_count += thr_eagain_count; } - - #define INQ_REPLY_LEN 96 #define INQ_CMD_LEN 6 @@ -543,8 +526,7 @@ do_rd_inc_wr_twice(const char * dev_name, unsigned int lba, int block, * in b (up to m_blen bytes). Does not use O_EXCL flag. Returns 0 on success, * else -1 . */ static int -do_inquiry_prod_id(const char * dev_name, int block, int wait_ms, - unsigned int & ebusys, char * b, int b_mlen) +do_inquiry_prod_id(const char * dev_name, int block, char * b, int b_mlen) { int sg_fd, ok, ret; struct sg_io_hdr pt; @@ -557,16 +539,7 @@ do_inquiry_prod_id(const char * dev_name, int block, int wait_ms, if (! block) open_flags |= O_NONBLOCK; - while (((sg_fd = open(dev_name, open_flags)) < 0) && - (EBUSY == errno)) { - ++ebusys; - if (wait_ms > 0) - this_thread::sleep_for(milliseconds{wait_ms}); - else if (0 == wait_ms) - this_thread::yield(); - else if (-2 == wait_ms) - sleep(0); // process yield ?? - } + sg_fd = open(dev_name, open_flags); if (sg_fd < 0) { snprintf(ebuff, EBUFF_SZ, "do_inquiry_prod_id: error opening file: %s", dev_name); @@ -627,59 +600,37 @@ do_inquiry_prod_id(const char * dev_name, int block, int wait_ms, return ret; } -static void -work_thread(const char * dev_name, unsigned int lba, int id, int block, - int excl, int num, int wait_ms) -{ - unsigned int thr_odd_count = 0; - unsigned int thr_ebusy_count = 0; - unsigned int thr_eagain_count = 0; - int k, res; - - console_mutex.lock(); - cerr << "Enter work_thread id=" << id << " excl=" << excl << " block=" - << block << endl; - console_mutex.unlock(); - for (k = 0; k < num; ++k) { - res = do_rd_inc_wr_twice(dev_name, lba, block, excl, wait_ms, k, - thr_ebusy_count, thr_eagain_count); - if (res < 0) - break; - if (res) - ++thr_odd_count; - } - console_mutex.lock(); - if (k < num) - cerr << "thread id=" << id << " FAILed at iteration: " << k << '\n'; - else - cerr << "thread id=" << id << " normal exit" << '\n'; - console_mutex.unlock(); - - odd_count_mutex.lock(); - odd_count += thr_odd_count; - ebusy_count += thr_ebusy_count; - eagain_count += thr_eagain_count; - odd_count_mutex.unlock(); -} - int main(int argc, char * argv[]) { - int k, res; - int block = 0; + int k, n, res; int force = 0; - unsigned int lba = DEF_LBA; - int num_per_thread = DEF_NUM_PER_THREAD; + int64_t ll; + unsigned int inq_ebusy_count = 0; int num_threads = DEF_NUM_THREADS; - int wait_ms = DEF_WAIT_MS; - int no_o_excl = 0; - char * dev_name = NULL; char b[64]; + struct timespec start_tm, end_tm; + struct opts_t opts; + struct opts_t * op; + + op = &opts; + op->dev_name = NULL; + op->direct = !! DEF_DIRECT; + op->lba = DEF_LBA; + op->lb_sz = DEF_LB_SZ;; + op->num_per_thread = DEF_NUM_PER_THREAD; + op->no_xfer = !! DEF_NO_XFER; + op->verbose = 0; + op->wait_ms = DEF_WAIT_MS; + op->c2e = SCSI_TUR; + op->bqd = BQ_DEFAULT; + op->block = !! DEF_BLOCKING; + page_size = sysconf(_SC_PAGESIZE); for (k = 1; k < argc; ++k) { - if (0 == memcmp("-b", argv[k], 2)) - ++block; + if (0 == memcmp("-d", argv[k], 2)) + op->direct = true; else if (0 == memcmp("-f", argv[k], 2)) ++force; else if (0 == memcmp("-h", argv[k], 2)) { @@ -687,100 +638,130 @@ main(int argc, char * argv[]) return 0; } else if (0 == memcmp("-l", argv[k], 2)) { ++k; - if ((k < argc) && isdigit(*argv[k])) - lba = (unsigned int)atoi(argv[k]); - else + if ((k < argc) && isdigit(*argv[k])) { + ll = sg_get_llnum(argv[k]); + if (-1 == ll) { + fprintf(stderr, "could not decode lba\n"); + return 1; + } else + op->lba = (uint64_t)ll; + } else break; } else if (0 == memcmp("-n", argv[k], 2)) { ++k; if ((k < argc) && isdigit(*argv[k])) - num_per_thread = atoi(argv[k]); + op->num_per_thread = atoi(argv[k]); else break; + } else if (0 == memcmp("-N", argv[k], 2)) + op->no_xfer = true; + else if (0 == memcmp("-q", argv[k], 2)) { + ++k; + if ((k < argc) && isdigit(*argv[k])) { + n = atoi(argv[k]); + if (0 == n) + op->bqd = BQ_AT_HEAD; + else if (1 == n) + op->bqd = BQ_AT_TAIL; + } + } else if (0 == memcmp("-R", argv[k], 2)) + op->c2e = SCSI_READ16; + else if (0 == memcmp("-s", argv[k], 2)) { + ++k; + if ((k < argc) && isdigit(*argv[k])) { + op->lb_sz = atoi(argv[k]); + if (op->lb_sz < 256) { + cerr << "Strange lb_sz, using 256" << endl; + op->lb_sz = 256; + } + } else + break; } else if (0 == memcmp("-t", argv[k], 2)) { ++k; if ((k < argc) && isdigit(*argv[k])) num_threads = atoi(argv[k]); else break; - } else if (0 == memcmp("-V", argv[k], 2)) { + } else if (0 == memcmp("-T", argv[k], 2)) + op->c2e = SCSI_TUR; + else if (0 == memcmp("-vvvv", argv[k], 5)) + op->verbose += 4; + else if (0 == memcmp("-vvv", argv[k], 4)) + op->verbose += 3; + else if (0 == memcmp("-vv", argv[k], 3)) + op->verbose += 2; + else if (0 == memcmp("-v", argv[k], 2)) + ++op->verbose; + else if (0 == memcmp("-V", argv[k], 2)) { printf("%s version: %s\n", util_name, version_str); return 0; } else if (0 == memcmp("-w", argv[k], 2)) { ++k; if ((k < argc) && (isdigit(*argv[k]) || ('-' == *argv[k]))) { if ('-' == *argv[k]) - wait_ms = - atoi(argv[k] + 1); + op->wait_ms = - atoi(argv[k] + 1); else - wait_ms = atoi(argv[k]); + op->wait_ms = atoi(argv[k]); } else break; - } else if (0 == memcmp("-xxx", argv[k], 4)) - no_o_excl += 3; - else if (0 == memcmp("-xx", argv[k], 3)) - no_o_excl += 2; - else if (0 == memcmp("-x", argv[k], 2)) - ++no_o_excl; + } else if (0 == memcmp("-W", argv[k], 2)) + op->c2e = SCSI_WRITE16; else if (*argv[k] == '-') { printf("Unrecognized switch: %s\n", argv[k]); - dev_name = NULL; + op->dev_name = NULL; break; } - else if (! dev_name) - dev_name = argv[k]; + else if (! op->dev_name) + op->dev_name = argv[k]; else { printf("too many arguments\n"); - dev_name = 0; + op->dev_name = NULL; break; } } - if (0 == dev_name) { + if (0 == op->dev_name) { usage(); return 1; } try { struct stat a_stat; - if (stat(dev_name, &a_stat) < 0) { + if (stat(op->dev_name, &a_stat) < 0) { perror("stat() on dev_name failed"); return 1; } if (! S_ISCHR(a_stat.st_mode)) { fprintf(stderr, "%s should be a sg device which is a char " - "device. %s\n", dev_name, dev_name); + "device. %s\n", op->dev_name, op->dev_name); fprintf(stderr, "is not a char device and damage could be done " "if it is a BLOCK\ndevice, exiting ...\n"); return 1; } if (! force) { - res = do_inquiry_prod_id(dev_name, block, wait_ms, ebusy_count, - b, sizeof(b)); + res = do_inquiry_prod_id(op->dev_name, op->block, b, sizeof(b)); if (res) { - fprintf(stderr, "INQUIRY failed on %s\n", dev_name); + fprintf(stderr, "INQUIRY failed on %s\n", op->dev_name); return 1; } // For safety, since written to, only permit scsi_debug // devices. Bypass this with '-f' option. if (0 != memcmp("scsi_debug", b, 10)) { - fprintf(stderr, "Since this utility writes to LBA %d, only " - "devices with scsi_debug\nproduct ID accepted.\n", - lba); + fprintf(stderr, "Since this utility writes to LBA 0x%" PRIx64 + ", only devices with scsi_debug\n" + "product ID accepted\n", op->lba); return 2; } + ebusy_count += inq_ebusy_count; } + start_tm.tv_sec = 0; + start_tm.tv_nsec = 0; + if (clock_gettime(CLOCK_MONOTONIC, &start_tm) < 0) + perror("clock_gettime failed"); vector vt; for (k = 0; k < num_threads; ++k) { - int excl = 1; - - if (no_o_excl > 1) - excl = 0; - else if ((0 == k) && (1 == no_o_excl)) - excl = 0; - - thread * tp = new thread {work_thread, dev_name, lba, k, block, - excl, num_per_thread, wait_ms}; + thread * tp = new thread {work_thread, k, op}; vt.push_back(tp); } @@ -791,13 +772,35 @@ main(int argc, char * argv[]) for (k = 0; k < (int)vt.size(); ++k) delete vt[k]; - if (no_o_excl) - cout << "Odd count: " << odd_count << endl; - else - cout << "Expecting odd count of 0, got " << odd_count << endl; - cout << "Number of EBUSYs: " << ebusy_count << endl; - cout << "Number of EAGAINs: " << eagain_count << endl; + n = uniq_pack_id.load() - 1; + if ((n > 0) && (0 == clock_gettime(CLOCK_MONOTONIC, &end_tm))) { + struct timespec res_tm; + double a, b; + res_tm.tv_sec = end_tm.tv_sec - start_tm.tv_sec; + res_tm.tv_nsec = end_tm.tv_nsec - start_tm.tv_nsec; + if (res_tm.tv_nsec < 0) { + --res_tm.tv_sec; + res_tm.tv_nsec += 1000000000; + } + a = res_tm.tv_sec; + a += (0.000001 * (res_tm.tv_nsec / 1000)); + b = (double)n; + if (a > 0.000001) { + printf("Time to complete %d commands was %d.%06d seconds\n", + n, (int)res_tm.tv_sec, (int)(res_tm.tv_nsec / 1000)); + cout << "Implies " << (b / a) << " IOPS" << endl; + } + } + + if (op->verbose) { + cout << "Number of async_starts: " << async_starts.load() << endl; + cout << "Number of async_finishes: " << async_finishes.load() << + endl; + cout << "Last pack_id: " << n << endl; + cout << "Number of EBUSYs: " << ebusy_count.load() << endl; + cout << "Number of EAGAINs: " << eagain_count.load() << endl; + } } catch(system_error& e) { cerr << "got a system_error exception: " << e.what() << '\n'; diff --git a/examples/sg_tst_excl.cpp b/examples/sg_tst_excl.cpp index b19d4a21..a4a85121 100644 --- a/examples/sg_tst_excl.cpp +++ b/examples/sg_tst_excl.cpp @@ -46,7 +46,7 @@ #include "sg_lib.h" #include "sg_io_linux.h" -static const char * version_str = "1.07 20131110"; +static const char * version_str = "1.08 20140710"; static const char * util_name = "sg_tst_excl"; /* This is a test program for checking O_EXCL on open() works. It uses @@ -72,6 +72,8 @@ static const char * util_name = "sg_tst_excl"; * g++ -Wall -std=c++11 -pthread -I ../include ../lib/sg_lib.o * ../lib/sg_lib_data.o ../lib/sg_io_linux.o -o sg_tst_excl * sg_tst_excl.cpp + * or use the C++ Makefile in that directory: + * make -f Makefile.cplus sg_tst_excl * * Currently this utility is Linux only and assumes the SG_IO v3 interface * which is supported by sg and block devices (but not bsg devices which diff --git a/examples/sgq_dd.c b/examples/sgq_dd.c index 7ed18a13..55b03c34 100644 --- a/examples/sgq_dd.c +++ b/examples/sgq_dd.c @@ -9,11 +9,11 @@ #include #include #include +#include #include #include #include #include -#include #include #include typedef unsigned char u_char; /* horrible, for scsi.h */ -- cgit v1.2.3