sdcard: generalize the IP, instantiate for ESP32 SPI channel too

fpga/iodevs.vh

@@ -2,30 +2,36 @@
 	wire [15:0] iodev_valid = iodev_mem_valid << cpu_mem_addr[10:7];
 
 	wire [31:0] iodev_rdata_led;
-	wire [0:0] iodev_valid_led = iodev_valid[0:0];
-	tri1 [0:0] iodev_wait_n_led;
+	wire [ 0:0] iodev_valid_led = iodev_valid[0:0];
+	tri1 [ 0:0] iodev_wait_n_led;
 
 	wire [31:0] iodev_rdata_reset;
-	wire [0:0] iodev_valid_reset = iodev_valid[1:1];
-	tri1 [0:0] iodev_wait_n_reset;
+	wire [ 0:0] iodev_valid_reset = iodev_valid[1:1];
+	tri1 [ 0:0] iodev_wait_n_reset;
 
 	wire [31:0] iodev_rdata_romcopy;
-	wire [0:0] iodev_valid_romcopy = iodev_valid[2:2];
-	tri1 [0:0] iodev_wait_n_romcopy;
+	wire [ 0:0] iodev_valid_romcopy = iodev_valid[2:2];
+	tri1 [ 0:0] iodev_wait_n_romcopy;
+
+	wire [31:0] iodev_rdata_sysclock;
+	wire [ 0:0] iodev_irq_sysclock;
+	wire [ 0:0] iodev_valid_sysclock = iodev_valid[3:3];
+	tri1 [ 0:0] iodev_wait_n_sysclock;
 
 	wire [31:0] iodev_rdata_console;
 	wire [ 0:0] iodev_irq_console;
-	wire [0:0] iodev_valid_console = iodev_valid[3:3];
-	tri1 [0:0] iodev_wait_n_console;
+	wire [ 0:0] iodev_valid_console = iodev_valid[4:4];
+	tri1 [ 0:0] iodev_wait_n_console;
 
 	wire [31:0] iodev_rdata_sdcard;
-	wire [0:0] iodev_valid_sdcard = iodev_valid[4:4];
-	tri1 [0:0] iodev_wait_n_sdcard;
+	wire [ 0:0] iodev_irq_sdcard;
+	wire [ 0:0] iodev_valid_sdcard = iodev_valid[5:5];
+	tri1 [ 0:0] iodev_wait_n_sdcard;
 
-	wire [31:0] iodev_rdata_sysclock;
-	wire [ 0:0] iodev_irq_sysclock;
-	wire [0:0] iodev_valid_sysclock = iodev_valid[5:5];
-	tri1 [0:0] iodev_wait_n_sysclock;
+	wire [31:0] iodev_rdata_esp;
+	wire [ 0:0] iodev_irq_esp;
+	wire [ 0:0] iodev_valid_esp = iodev_valid[6:6];
+	tri1 [ 0:0] iodev_wait_n_esp;
 
 	// I/O input MUX
 	always @(*)
@@ -33,22 +39,26 @@
 			4'd0:	 iodev_rdata = iodev_rdata_led;
 			4'd1:	 iodev_rdata = iodev_rdata_reset;
 			4'd2:	 iodev_rdata = iodev_rdata_romcopy;
-			4'd3:	 iodev_rdata = iodev_rdata_console;
-			4'd4:	 iodev_rdata = iodev_rdata_sdcard;
-			4'd5:	 iodev_rdata = iodev_rdata_sysclock;
+			4'd3:	 iodev_rdata = iodev_rdata_sysclock;
+			4'd4:	 iodev_rdata = iodev_rdata_console;
+			4'd5:	 iodev_rdata = iodev_rdata_sdcard;
+			4'd6:	 iodev_rdata = iodev_rdata_esp;
 			default: iodev_rdata = 32'hffffffff;
 		endcase
 
 	tri0 [31:0] sys_irq;
-	assign sys_irq[ 3] = iodev_irq_console[0];
-	assign sys_irq[ 4] = iodev_irq_sysclock[0];
+	assign sys_irq[ 3] = iodev_irq_sysclock[0];
+	assign sys_irq[ 4] = iodev_irq_console[0];
+	assign sys_irq[ 5] = iodev_irq_sdcard[0];
+	assign sys_irq[ 6] = iodev_irq_esp[0];
 
-	localparam [31:0] irq_edge_mask =  32'h00000010;
-	localparam [31:0] irq_masked    = ~32'h0000001f;
+	localparam [31:0] irq_edge_mask =  32'h00000008;
+	localparam [31:0] irq_masked    = ~32'h0000007f;
 
 	wire iodev_wait_n = (&iodev_wait_n_led) & 
 		(&iodev_wait_n_reset) & 
 		(&iodev_wait_n_romcopy) & 
+		(&iodev_wait_n_sysclock) & 
 		(&iodev_wait_n_console) & 
 		(&iodev_wait_n_sdcard) & 
-		(&iodev_wait_n_sysclock);
+		(&iodev_wait_n_esp);

fpga/max80.sv

@@ -386,7 +386,7 @@ module max80 (
    // I/O device map from iodevs.conf
    wire        iodev_mem_valid = cpu_mem_quad[3];
 `include "iodevs.vh"
-   
+
    //
    // SDRAM
    //
@@ -446,16 +446,11 @@ module max80 (
 		.wacc2    ( sdram_rom_wacc )
 		);
 
-   // SPI bus (free for ESP32)
-   assign spi_clk        = 1'bz;
-   assign spi_miso       = 1'bz;
-   assign spi_mosi       = 1'bz;
-   assign spi_cs_esp_n   = 1'bz;
-   assign spi_cs_flash_n = 1'bz;
 
    // ESP32
-   assign esp_io0  = 1'bz;
-   assign esp_int  = 1'bz;
+   assign spi_cs_flash_n = 1'bz;
+   assign esp_io0  = 1'b1;	 // If pulled down on reset, ESP32 will enter
+				 // firmware download mode
 
    // I2C
    assign i2c_scl = 1'bz;
@@ -597,7 +592,7 @@ module max80 (
 
    // Register I/O data to reduce the size of the read data MUX
    reg [31:0]  iodev_rdata_q;
-   
+
    // Read data MUX
    always @(*)
      case ( cpu_mem_quad )
@@ -649,7 +644,7 @@ module max80 (
      rom_done_q <= rom_done;
 
    assign iodev_rdata_romcopy = { 31'b0, rom_done_q };
-   
+
    //
    // Serial port. Direct to the CP2102N for reworked
    // boards or to GPIO for non-reworked boards, depending on
@@ -708,24 +703,56 @@ module max80 (
 
 
    // SD card
-   sdcard sdcard (
-		  .rst_n   ( rst_n ),
-		  .clk     ( sys_clk ),
-		  .sd_cs_n ( sd_dat[3] ),
-		  .sd_di   ( sd_cmd ),
-		  .sd_sclk ( sd_clk ),
-		  .sd_do   ( sd_dat[0] ),
-		  .sd_cd_n ( 1'b0 ),
-
-		  .wdata   ( cpu_mem_wdata ),
-		  .rdata   ( iodev_rdata_sdcard ),
-		  .valid   ( iodev_valid_sdcard ),
-		  .wstrb   ( cpu_mem_wstrb ),
-		  .addr    ( cpu_mem_addr[6:2] ),
-		  .wait_n  ( iodev_wait_n_sdcard )
-		  );
+   sdcard #(
+	    .with_irq_mask ( 8'b0000_0001 )
+	    )
+   sdcard (
+	   .rst_n   ( rst_n ),
+	   .clk     ( sys_clk ),
+	   .sd_cs_n ( sd_dat[3] ),
+	   .sd_di   ( sd_cmd ),
+	   .sd_sclk ( sd_clk ),
+	   .sd_do   ( sd_dat[0] ),
+	   .sd_cd_n ( 1'b0 ),
+	   .sd_irq_n ( 1'b1 ),
+
+	   .wdata   ( cpu_mem_wdata ),
+	   .rdata   ( iodev_rdata_sdcard ),
+	   .valid   ( iodev_valid_sdcard ),
+	   .wstrb   ( cpu_mem_wstrb ),
+	   .addr    ( cpu_mem_addr[6:2] ),
+	   .wait_n  ( iodev_wait_n_sdcard ),
+	   .irq     ( iodev_irq_sdcard )
+	   );
    assign sd_dat[2:1] = 2'bzz;
 
+   // SPI bus to ESP32; using the sdcard IP as a SPI master for now at
+   // least...
+   sdcard #(
+	    .with_irq_mask ( 8'b0000_0101 ),
+	    .with_crc7     ( 1'b0 ),
+	    .with_crc16    ( 1'b0 )
+	    )
+   esp (
+	.rst_n    ( rst_n ),
+
+	.clk      ( sys_clk ),
+	.sd_cs_n  ( spi_cs_esp_n ),
+	.sd_di    ( spi_mosi ),
+	.sd_sclk  ( spi_clk ),
+	.sd_do    ( spi_miso ),
+	.sd_cd_n  ( 1'b0 ),
+	.sd_irq_n ( esp_int ),
+
+	.wdata  ( cpu_mem_wdata ),
+	.rdata  ( iodev_rdata_esp ),
+	.valid  ( iodev_valid_esp ),
+	.wstrb  ( cpu_mem_wstrb ),
+	.addr   ( cpu_mem_addr[6:2] ),
+	.wait_n ( iodev_wait_n_esp ),
+	.irq	( iodev_irq_esp )
+	);
+
    // System local clock (not an RTC, but settable from one)
    // Also provides a periodic interrupt (set to 32 Hz)
    // XXX: the RTC 32 kHz signal is missing a pull-up,

fpga/sdcard.sv

@@ -16,24 +16,35 @@
 // This runs the SD card in SPI mode. In the future, consider improving
 // performance by switching to quad SD mode.
 //
+// Note: this is also usable as generic SPI master in many cases.
 
-module sdcard (
-	       input 		 rst_n, // Global reset
-	       input 		 clk, // System clock (84 MHz)
+module sdcard
+ #(
+   parameter [0:0]  with_crc7     = 1'b1,
+   parameter [6:0]  crc7_poly     = 7'b000_1001,
+   parameter [0:0]  with_crc16    = 1'b1,
+   parameter [15:0] crc16_poly    = 16'b0001_0000_0010_0001,
+   parameter [7:0]  with_irq_mask = 8'b0000_0000
+   )
+   (
+    input	      rst_n,    // Global reset
+    input	      clk,      // System clock (84 MHz)
 
-	       output 		 sd_cs_n, // SD card CS# (CD, DAT3)
-	       output 		 sd_di, // SD card DI (MOSI, CMD)
-	       output 		 sd_sclk, // SD card CLK (SCLK)
-	       input 		 sd_do, // SD card SO (MISO, DAT0)
-	       input 		 sd_cd_n, // Card detect
+    output	      sd_cs_n,  // SD card CS# (CD, DAT3)
+    output	      sd_di,    // SD card DI (MOSI, CMD)
+    output	      sd_sclk,  // SD card CLK (SCLK)
+    input	      sd_do,    // SD card SO (MISO, DAT0)
+    input	      sd_cd_n,  // Card detect
+    input	      sd_irq_n, // External IRQ input (optional)
 
-	       input [31:0] 	 wdata, // CPU data out (CPU->controller)
-	       output reg [31:0] rdata, // CPU data in (controller->CPU)
-	       input 		 valid, // Memory valid
-	       input [3:0] 	 wstrb, // Write strobes
-	       input [4:0] 	 addr, // Address bits
-	       output 		 wait_n	// Hold mem_ready
-	       );
+    input [31:0]      wdata,    // CPU data out (CPU->controller)
+    output reg [31:0] rdata,    // CPU data in (controller->CPU)
+    input	      valid,    // Memory valid
+    input [3:0]       wstrb,    // Write strobes
+    input [4:0]       addr,     // Address bits
+    output	      wait_n,   // Hold mem_ready
+    output	      irq	// CPU interrupt request
+    );
 
    // ------------------------------------------------------------------------
    //  SD card interface
@@ -48,10 +59,12 @@ module sdcard (
    //
    //  Write:
    //  00000	- control register:
-   //             [6:0] - speed divider (CPU_HZ/(2*(divider+1)))
-   //                 7 - CS# active
-   //                 8 - clear read CRC registers
-   //		      9 - clear write CRC registers
+   //              [6:0] - speed divider (CPU_HZ/(2*(divider+1)))
+   //                [7] - CS# active
+   //             [15:8] - IRQ enable mask
+   //               [22] - clear read CRC
+   //               [23] - clear write CRC
+   //
    //  x0xxx	- reserved
    //  x1e00     - load shift register but don't start transaction
    //  x1e01     - load shift register and start transaction, 8 bits
@@ -65,10 +78,12 @@ module sdcard (
    //  bytes within a dword); the input latch right-aligned.
    //
    //  Read:
-   //  00000	- control register
-   //  00010	-   [7:0] - read CRC7  + final 1 bit
+   //  00000	-  [15:0] - control register
+   //                [16] - busy status
+   //             [31:24] - IRQ status
+   //  00100	-   [7:0] - read CRC7  + final 1 bit
    //             [31:16] - read CRC16
-   //  00011	-   [7:0] - write CRC7 + final 1 bit
+   //  00101	-   [7:0] - write CRC7 + final 1 bit
    //             [31:16] - write CRC16
    //  x0xxx    - reserved
    //  x1e00    - read shift register but don't start transaction
@@ -77,6 +92,13 @@ module sdcard (
    //  x1e11    - read shift register and start transaction, 32 bits
    //  11xxx    - clear read CRC registers
    //
+   //  Addresses of the form 000xx are non-blocking; others stall the CPU
+   //  until the current transaction is complete.
+   //
+   //  Available interrupts are:
+   //  0 - unit idle
+   //  1 - card detect (sd_cd_n low)
+   //  2 - external interrupt (sd_irq_n low)
    // ------------------------------------------------------------------------
 
    reg [31:0] sd_shr_out;
@@ -87,8 +109,6 @@ module sdcard (
    reg	      sd_active_neg;	// Transfer in progress, first pos clock seen
    reg	      sd_crcstb;	// Strobe for CRC generator
    reg	      sd_cs_reg;	// CS# active (positive logic, so inverted)
-   wire       sd_cmd = valid & ~sd_active; // CPU command we can act on
-   reg	      sd_cmd_ok;	// Valid CPU command received
    wire       sd_data_out = sd_shr_out[31];
    reg	      sd_clk_out;	// Output clock signal
 
@@ -107,14 +127,23 @@ module sdcard (
    // sd_cmd:     0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0
    // sd_cmd_ok:  0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 1 1 0 0
    // cpu_wait_n: 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 1 1 1 1 1
+   //
+
+   wire       valid_blocking = valid & (addr[4:2] != 3'b000);
+   wire       sd_cmd = valid_blocking & ~sd_active;
+   // CPU command we can act on
+   reg	      sd_cmd_ok;	// Valid CPU command received
 
    always @(negedge rst_n or posedge clk)
      if (~rst_n)
        sd_cmd_ok <= 1'b0;
      else
-       sd_cmd_ok <= valid & (~sd_active | sd_cmd_ok);
+       sd_cmd_ok <= valid_blocking & (~sd_active | sd_cmd_ok);
 
-   assign wait_n = ~(valid & sd_active) | sd_cmd_ok;
+   assign wait_n = ~(valid_blocking & sd_active) | sd_cmd_ok;
+
+   // Valid *nonblocking* command
+   wire       sd_cmd_nonblock = valid & (addr[4:2] == 3'b000);
 
    // SD clock generator; this counter is used to generate the slow clock.
    reg [6:0]  sd_clk_div;
@@ -149,6 +178,25 @@ module sdcard (
      else
        sd_clk_out <= (sd_clk_out | sd_clk_pos) & ~sd_clk_neg;
 
+   // IRQ handling (extensible for future uses)
+   reg [7:0] irq_status;
+   always @(posedge clk)
+     begin
+	irq_status <= with_irq_mask &
+		      {
+		       5'b0, // Reserved for future uses
+		       ~sd_irq_n,
+		       ~sd_cd_n,
+		       ~sd_active
+		       };
+     end
+   reg [7:0] irq_en;
+
+   assign     irq = |(irq_status & irq_en & with_irq_mask);
+
+   //
+   // Main shift register state machine
+   //
    reg [1:0]  clear_crc;
 
    always @(negedge rst_n or posedge clk)
@@ -164,6 +212,7 @@ module sdcard (
 	  sd_shr_in     <= 32'hffff_ffff;
 	  sd_shr_in_q   <= 32'hffff_ffff;
 	  clear_crc     <= 2'b11;
+	  irq_en        <= 8'b0;
        end
      else
        begin
@@ -185,11 +234,23 @@ module sdcard (
 	  clear_crc <= 2'b00;	   // No clearing by default
 	  sd_crcstb <= sd_clk_pos; // CRCs are computed one cycle after posedge
 
+	  if (sd_cmd_nonblock)
+	    casez(addr[1:0])
+	      2'b00: begin
+		 if (wstrb[0]) {sd_cs_reg, sd_clk_div} <= wdata[7:0];
+		 if (wstrb[1]) irq_en <= wdata[15:8] & with_irq_mask;
+		 if (wstrb[2]) clear_crc <= wdata[23:22];
+	      end
+	      default: begin
+		 // Do nothing
+	      end
+	    endcase
+
 	  if (sd_cmd)
 	    begin
 	       if (addr[4:3] == 2'b11)
 		 clear_crc <= {|wstrb, ~|wstrb};
-	       
+
 	       casez (addr)
 		 5'b?10??: begin
 		    // Load in host (littleendian) byte order
@@ -205,10 +266,6 @@ module sdcard (
 		    if (wstrb[1]) sd_shr_out[15: 8] <= wdata[15: 8];
 		    if (wstrb[0]) sd_shr_out[ 7: 0] <= wdata[ 7: 0];
 		 end
-		 5'b00000: begin
-		    if (wstrb[0]) {sd_cs_reg, sd_clk_div} <= wdata[7:0];
-		    if (wstrb[1]) clear_crc <= wdata[9:8];
-		 end
 		 default: begin
 		    // do nothing
 		 end
@@ -251,39 +308,62 @@ module sdcard (
    reg  [6:0] sd_crc7 [0:1];	// CRC-7 shift register
    reg [15:0] sd_crc16[0:1];	// CRC-16 shift register
 
-   localparam  [6:0] crc7_poly  = 7'b000_1001;
-   localparam [15:0] crc16_poly = 16'b0001_0000_0010_0001;
-   
-   always @(posedge clk)
-     for (int i = 0; i < 2; i = i+1)
-       begin
-	  if (clear_crc[i])
-	    begin
-	       sd_crc7[i]  <= 7'h00;
-	       sd_crc16[i] <= 16'h0000;
-	    end
-	  else if (sd_crcstb)
-	    begin
-	       sd_crc7[i]  <= { sd_crc7[i][5:0], 1'b0 }
-			      ^ ({7{sd_crcbit[i] ^ sd_crc7[i][6]}}
-				 & crc7_poly);
-	       sd_crc16[i] <= { sd_crc16[i][14:0], 1'b0 }
-			      ^ ({16{sd_crcbit[i] ^ sd_crc16[i][15]}}
-				 & crc16_poly);
-	    end // else: !if(clear_crc[i])
-       end // for (int i = 0; i < 2; i = i+1)
+   always @(negedge rst_n or posedge clk)
+     if (~rst_n)
+       for (int i = 0; i < 2; i = i+1)
+	 begin
+	    sd_crc7[i]  <= 7'hxx;
+	    sd_crc16[i] <= 16'hxxxx;
+	 end
+     else
+       for (int i = 0; i < 2; i = i+1)
+	 begin
+	    if (clear_crc[i])
+	      begin
+		 sd_crc7[i]  <= 7'h00;
+		 sd_crc16[i] <= 16'h0000;
+	      end
+	    else if (sd_crcstb)
+	      begin
+		 if (with_crc7)
+		   sd_crc7[i]  <= { sd_crc7[i][5:0], 1'b0 }
+				  ^ ({7{sd_crcbit[i] ^ sd_crc7[i][6]}}
+				     & crc7_poly);
+		 if (with_crc16)
+		   sd_crc16[i] <= { sd_crc16[i][14:0], 1'b0 }
+				  ^ ({16{sd_crcbit[i] ^ sd_crc16[i][15]}}
+				     & crc16_poly);
+	      end // else: !if(clear_crc[i])
+	 end // for (int i = 0; i < 2; i = i+1)
 
    // Data out MUX
-   always @(*)
+   always_comb
      begin
 	casez (addr)
-	  5'b0_0000: rdata = { 24'b0, sd_cs_reg, sd_clk_div };
-	  5'b0_0010: rdata = { sd_crc16[0], 8'b0, sd_crc7[0], 1'b1 };
-	  5'b0_0011: rdata = { sd_crc16[1], 8'b0, sd_crc7[1], 1'b1 };
-	  5'b?_10??: rdata = { sd_shr_in_q[7:0], sd_shr_in_q[15:8],
-			       sd_shr_in_q[23:16], sd_shr_in_q[31:24] };
-	  5'b?_11??: rdata = sd_shr_in_q;
-	  default:   rdata = 32'hxxxx_xxxx;
+	  5'b0_0000: begin
+	     rdata[31:16] = { irq_status, 7'b0, sd_active };
+	     rdata[15: 0] = { irq_en, sd_cs_reg, sd_clk_div };
+	  end
+	  5'b0_0100: begin
+	     rdata[31:16] = with_crc16 ? sd_crc16[0] : 16'b0;
+	     rdata[15: 8] = 8'b0;
+	     rdata[ 7: 0] = with_crc7  ? { sd_crc7[0], 1'b1 } : 8'b0;
+	  end
+	  5'b0_0101: begin
+	     rdata[31:16] = with_crc16 ? sd_crc16[1] : 16'b0;
+	     rdata[15: 8] = 8'b0;
+	     rdata[ 7: 0] = with_crc7  ? { sd_crc7[1], 1'b1 } : 8'b0;
+	  end
+	  5'b?_10??: begin
+	     rdata = { sd_shr_in_q[7:0], sd_shr_in_q[15:8],
+		       sd_shr_in_q[23:16], sd_shr_in_q[31:24] };
+	  end
+	  5'b?_11??: begin
+	     rdata = sd_shr_in_q;
+	  end
+	  default: begin
+	     rdata = 32'hxxxx_xxxx;
+	  end
 	endcase
      end
 

fw/ioregs.h

@@ -67,10 +67,10 @@
 #define SDCARD_CTL		IODEVL(SDCARD_DEV,0)
 #define SDCARD_CTL_SPEED	IODEVB0(SDCARD_DEV,0)
 #define SDCARD_CTL_CLRCRC	IODEVB1(SDCARD_DEV,0)
-#define SDCARD_CRC7_RD		IODEVRB0(SDCARD_DEV,2)
-#define SDCARD_CRC16_RD		IODEVRH1(SDCARD_DEV,2)
-#define SDCARD_CRC7_WR		IODEVRB0(SDCARD_DEV,3)
-#define SDCARD_CRC16_WR		IODEVRH1(SDCARD_DEV,3)
+#define SDCARD_CRC7_RD		IODEVRB0(SDCARD_DEV,4)
+#define SDCARD_CRC16_RD		IODEVRH1(SDCARD_DEV,4)
+#define SDCARD_CRC7_WR		IODEVRB0(SDCARD_DEV,5)
+#define SDCARD_CRC16_WR		IODEVRH1(SDCARD_DEV,5)
 
 /* Speed values, not including -1 adjustment */
 #define SD_SLOW		128	/* 328 kHz */

iodevs.conf

@@ -15,7 +15,8 @@ our @iodevs = (
     { -name => 'led' },
     { -name => 'reset' },
     { -name => 'romcopy' },
+    { -name => 'sysclock', -irq => 'e' },
     { -name => 'console',  -irq => 'l' },
-    { -name => 'sdcard' },
-    { -name => 'sysclock', -irq => 'e' }
+    { -name => 'sdcard',   -irq => 'l' },
+    { -name => 'esp',      -irq => 'l' }
 );

tools/iodevs.pl

@@ -144,10 +144,10 @@ sub generate_verilog($)
 		length($irq)-1, $name, $didx;
 	}
 
-	printf $out "\twire [%d:0] iodev_valid_%s = iodev_valid[%d:%d];\n",
+	printf $out "\twire [%2d:0] iodev_valid_%s = iodev_valid[%d:%d];\n",
 	    $dcount-1, $name, $ndev+$dcount-1, $ndev;
 
-	printf $out "\ttri1 [%d:0] iodev_wait_n_%s;\n", $dcount-1, $name;
+	printf $out "\ttri1 [%2d:0] iodev_wait_n_%s;\n", $dcount-1, $name;
 	push(@wait, "(&iodev_wait_n_$name)");
 
 	print $out "\n";