flashesp: include all ESP32 flash data in .fw; script to flash

Add all ESP32 flash data (partition table, boot loader, and OTA
control partition) in the .fw image; this data is extremely
compressible so it adds very little to the size.

Add a Perl script (esp32/flashesp.pl) to flash the .fw image to the
ESP using esptool.py over a serial port or USB.

common/fwimg.h

@@ -26,7 +26,10 @@ enum fw_data_type {
     FDT_TARGET,			/* Subsystem string (must match) */
     FDT_NOTE,			/* Version: XXXXX or similar */
     FDT_ESP_OTA,		/* ESP32 OTA image */
-    FDT_FPGA_INIT		/* FPGA bitstream for update */
+    FDT_FPGA_INIT,		/* FPGA bitstream for update */
+    FDT_ESP_PART,		/* ESP32 partition table */
+    FDT_ESP_SYS,		/* ESP32 boot loader, OTA control, etc */
+    FDT_ESP_TOOL		/* esptool.py options for serial flashing */
 };
 enum fw_data_flags {
     FDF_OPTIONAL     = 0x0001	/* Ignore if chunk data type unknown */

esp32/esptool.opt

@@ -0,0 +1 @@
+baud 921600 chip esp32s2 flash_mode dio flash_freq 80m flash_size 4MB

esp32/flashesp.pl

@@ -0,0 +1,182 @@
+#!/usr/bin/perl
+
+use strict;
+use integer;
+use PerlIO::gzip;
+use File::Temp;
+use File::Spec;
+
+my $esptool = ($^O eq 'MSWin32') ? 'esptool.exe' : 'esptool.py';
+$esptool = $ENV{'ESPTOOL'} || $esptool;
+
+my $FW_MAGIC = 0x7a07fbd6;
+
+my %datatypes = (
+    'end'       => 0,		# End of data
+    'data'      => 1,		# FPGA flash data
+    'target'    => 2,		# Firmware target string
+    'note'      => 3,		# Informative string
+    'espota'    => 4,		# ESP32 OTA image
+    'fpgainit'  => 5,		# FPGA bypass (transient) image during update
+    'esppart'   => 6,		# ESP32 partition table
+    'espsys'    => 7,		# ESP32 boot loader, OTA control partition...
+    'esptool'   => 8		# esptool.py options for flashing
+    );
+my @type;
+foreach my $t (keys(%datatypes)) {
+    $type[$datatypes{$t}] = $t;
+}
+
+my $FDF_OPTIONAL = 0x0001;
+
+my $STRING_MAX_LEN = 4095;
+
+sub getint($) {
+    my($s) = @_;
+
+    return undef
+	unless ($s =~ /^(([1-9][0-9]+)|(0(x[0-9a-f]+|[0-7]*)))([kmgtpe]?)$/i);
+
+    my $o = oct($3) + $2;
+    my $p = lc($5);
+
+    if ($p eq 'k') {
+	$o <<= 10;
+    } elsif ($p eq 'm') {
+	$o <<= 20;
+    } elsif ($p eq 'g') {
+	$o <<= 30;
+    } elsif ($p eq 't') {
+	$o <<= 40;
+    } elsif ($p eq 'p') {
+	$o <<= 50;
+    } elsif ($p eq 'e') {
+	$o <<= 60;
+    }
+    return $o;
+}
+
+sub filelen($) {
+    my($f) = @_;
+    my @s = stat($f);
+
+    return $s[7];
+}
+
+sub unquote_cmd($) {
+    my($s) = @_;
+    my @a;
+
+    $s =~ s/[\r\n]+/ /g;
+    while ($s =~ /^\s*(?:\"((?:[^\"]+|\"\")*)\"|(\S+))(\s.*)?$/) {
+	push(@a, $1.$2);
+	$s = $3;
+    }
+
+    return @a;
+}
+
+my @args = @ARGV;
+my $file = shift(@args);
+
+if (!defined($file)) {
+    die "Usage: $0 fwfile [esptool options...]\n";
+}
+
+open(my $fw, '<:gzip', $file)
+    or die "$0: $file: $!\n";
+
+my @chunks = ();
+
+my $err = 0;
+
+my $hdr;
+while (read($fw, $hdr, 16) == 16) {
+    # magic type flags data_len addr
+    my @h = unpack('VvvVV', $hdr);
+    my $c = { 'hdr' => $hdr, 'magic' => $h[0], 'type' => $h[1],
+		  'flags' => $h[2], 'len' => $h[3], 'addr' => $h[4] };
+    if ($c->{'magic'} != $FW_MAGIC) {
+	print STDERR "$0: $file: bad chunk magic\n";
+	$err = 1;
+	last;
+    }
+
+    my $t = $type[$c->{'type'}];
+    last if ($t eq 'end'); # End of stream
+
+    my $d;
+    if (read($fw, $d, $c->{'len'}) != $c->{'len'}) {
+	print STDERR "$0: $file: short chunk read\n";
+	$err = 1;
+	last;
+    }
+    $c->{'data'} = $d;
+
+    push(@chunks, $c);
+}
+
+close($fw);
+exit $err if ($err);
+
+my $td = File::Temp->newdir(CLEANUP => 1);
+
+my @espfiles = ();
+my %espopt = ('before' => 'default_reset', 'after' => 'hard_reset',
+	      'baud' => 115200, 'port' => undef, 'chip' => undef,
+	      'flash_mode' => undef, 'flash_freq' => undef,
+	      'flash_size' => undef);
+
+my $nc = 0;
+foreach my $c ( @chunks ) {
+    my $t = $type[$c->{'type'}];
+    if ($t eq 'esptool') {
+	my $s = $c->{'data'};
+	$s =~ s/[\r\n]+/ /g;
+	while ($s =~ /^\s*(\S+)\s+(\S+)(.*)/) {
+	    $espopt{$1} = $2;
+	    $s = $3;
+	}
+    } elsif ($t =~ /^esp/ && $c->{'addr'}) {
+	my $addr = sprintf('%x', $c->{'addr'});
+	my $tff = File::Spec->catfile($td, $t.$nc.'_'.$addr.'.bin');
+	open(my $tf, '>', $tff) or die "$0: $tff: $!\n";
+	print $tf $c->{'data'};
+	close($tf);
+	push(@espfiles, '0x'.$addr, $tff);
+    }
+    $nc++;
+}
+
+foreach my $e (keys %espopt) {
+    my $ev = $ENV{"ESP\U$e"};
+    if (defined($ev)) {
+	$espopt{$e} = $ev;
+    }
+}
+
+my @espcmd = unquote_cmd($esptool);
+foreach my $o (sort grep (!/^flash_/, keys %espopt)) {
+    if (defined($espopt{$o})) {
+	push(@espcmd, "--$o", $espopt{$o});
+    }
+}
+push(@espcmd, @args);
+push(@espcmd, 'write_flash', '-z');
+foreach my $o (sort grep (/^flash_/, keys %espopt)) {
+    if (defined($espopt{$o})) {
+	push(@espcmd, "--$o", $espopt{$o});
+    }
+}
+push(@espcmd, @espfiles);
+
+if (defined($ENV{'VERBOSE'})) {
+    print STDERR join(' ', @espcmd), "\n";
+}
+
+$err = system(@espcmd);
+if ($err == -1) {
+    print STDERR "$0: $espcmd[0]: $!\n";
+}
+
+exit !!$err;

fpga/Makefile

@@ -45,9 +45,18 @@ allout   = $(foreach o,$(alltarg),$(outdir)/$(1).$(o))
 
 sram_src = ../rv32/
 
-DRAM_ADDR  := 0x100000
-DRAM_IMAGE := ../rv32/dram.bin
-ESP_IMAGE  := ../esp32/output/max80.ino.bin
+DRAM_ADDR        := 0x100000
+DRAM_IMAGE       := ../rv32/dram.bin
+
+ESP_ADDR         := 0x10000
+ESP_IMAGE        := ../esp32/output/max80.ino.bin
+ESP_BOOT_ADDR    := 0x1000
+ESP_BOOT_IMAGE   := ../esp32/output/max80.ino.bootloader.bin
+ESP_PART_ADDR    := 0x8000
+ESP_PART_IMAGE   := ../esp32/output/max80.ino.partitions.bin
+ESP_OTACTL_ADDR  := 0xe000
+ESP_OTACTL_IMAGE := ../esp32/boot_app0.bin
+ESP_TOOL_OPT     := ../esp32/esptool.opt
 
 IDCODE     := 0x020f20dd		# JTAG IDCODE of FPGA
 
@@ -181,7 +190,11 @@ $(outdir)/%.fw: $(outdir)/%.rpf $(outdir)/bypass.rbf \
 		-type fpgainit -addr $(IDCODE) -optional -file $(outdir)/bypass.rbf \
 		-type data -addr 0 -file $< \
 		-type data -addr $(DRAM_ADDR) -file $(DRAM_IMAGE) \
-		-type espota -file $(ESP_IMAGE) \
+		-type esptool -optional -file $(ESP_TOOL_OPT) \
+		-type espota -addr $(ESP_ADDR) -file $(ESP_IMAGE) \
+		-type esppart -optional -addr $(ESP_PART_ADDR) -file $(ESP_PART_IMAGE) \
+		-type espsys -optional -addr $(ESP_BOOT_ADDR) -file $(ESP_BOOT_IMAGE) \
+		-type espsys -optional -addr $(ESP_OTACTL_ADDR) -file $(ESP_OTACTL_IMAGE) \
 	| $(GZIP) -9 > $@
 
 $(outdir)/%.update.svf: ./scripts/flashsvf.pl \

fpga/max80.qpf

@@ -19,12 +19,12 @@
 #
 # Quartus Prime
 # Version 21.1.0 Build 842 10/21/2021 SJ Lite Edition
-# Date created = 12:23:20  June 22, 2022
+# Date created = 15:52:01  June 22, 2022
 #
 # -------------------------------------------------------------------------- #
 
 QUARTUS_VERSION = "21.1"
-DATE = "12:23:20  June 22, 2022"
+DATE = "15:52:01  June 22, 2022"
 
 # Revisions
 

tools/mkfwimage.pl

@@ -2,7 +2,6 @@
 
 use strict;
 use integer;
-use Digest::MD5;
 
 my $FW_MAGIC = 0x7a07fbd6;
 
@@ -12,8 +11,15 @@ my %datatypes = (
     'target'    => 2,		# Firmware target string
     'note'      => 3,		# Informative string
     'espota'    => 4,		# ESP32 OTA image
-    'fpgainit'  => 5		# FPGA bypass (transient) image during update
+    'fpgainit'  => 5,		# FPGA bypass (transient) image during update
+    'esppart'   => 6,		# ESP32 partition table
+    'espsys'    => 7,		# ESP32 boot loader, OTA control partition...
+    'esptool'   => 8		# esptool.py options for flashing
     );
+my @type;
+foreach my $t (keys(%datatypes)) {
+    $type[$datatypes{$t}] = $t;
+}
 
 my $FDF_OPTIONAL = 0x0001;
 
@@ -55,8 +61,8 @@ sub output_chunk($$$) {
     print $out pack("VvvVV", $FW_MAGIC,
 		    $options->{'type'}, $options->{'flags'},
 		    length($data), $options->{'addr'});
-    printf STDERR "chunk: type %u flags 0x%x length %u addr 0x%x\n",
-	$options->{'type'}, $options->{'flags'},
+    printf STDERR "chunk: type %s (%u) flags 0x%x length %u addr 0x%x\n",
+	$type[$options->{'type'}], $options->{'type'}, $options->{'flags'},
 	length($data), $options->{'addr'};
     print $out $data;
 }