Hexdump utility in x86 NASM assembly

Question

It's a simple hexdump that prints to stdout. I wanted to handle correctly input coming from the user typing at the terminal, this made the logic a little more complicated than just exiting after reading less than requested. It doesn't use the stack or memory variables, except for the tables and buffers, so I employed every register freely.

I'm learning, so I would like for every possible improvement to be pointed out.

; Description     : Prints stdin to stdout as hex and displays ASCII next to it.
;                   It handles correctly the cases in which a sys_read returns
;                   less than was requested but an EOF was not received, this
;                   allows the user to dump arbitrary text from stdin to a file 
;                   in the following way: ./hexdump >> dump.txt
;
; Build using these commands:
;   nasm -f elf hexdump.asm
;   ld -o hexdump hexdump.o -m elf_i386 -s
;
; Usage:
;   ./hexdump << input_file

section .data
STDIN: equ 0
STDOUT: equ 1
STDERR: equ 2
EXIT_SUCCESS: equ 0
SYS_READ: equ 3
SYS_WRITE: equ 4
SYS_EXIT: equ 1
NEW_LINE: equ 0ah
AsciiTable: db '................................ !"',"#$%&'()*+,-./0123456789:;"
            db '<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz'
            db '{|}~...........................................................'
            db '...............................................................'
            db '.......'
align 4
Template: db '00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 '
DivPos:   db '| '
AsciiPart: db '................',NEW_LINE
TEMPLATELENGTH: equ $-Template
align 2
HexTable: db '000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2'
          db '02122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f40'
          db '4142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606'
          db '162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f8081'
          db '82838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a'
          db '2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2'
          db 'c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e'
          db '3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff'

section .bss
BUFFERSIZE: equ 160
Buffer: resb BUFFERSIZE

section .text

global _start ; make it visible

_start:
        xor esi,esi                       ; position in buffer
        xor ebp,ebp                       ; buffer size
        mov edi,Read                      ; address to jmp
        mov esp,BUFFERSIZE                ; how much to read
  Fill: mov eax,SYS_READ
        xor ebx,ebx                       ; STDIN
        lea ecx,[Buffer+ebp]              ; destination
        mov edx,esp                       ; size
        int 80h                           ; call
        cmp eax,0                         ; check return value
        jg Ok                             ; read something
        jl Err                            ; < 0 -> error
        jmp Last                          ; 0 = EOF

  Ok:   add ebp,eax                       ; update size
  Read: lea esp,[esi+16]                  ; where we'll be if we print
        sub esp,ebp                       ; how much we have to read to reach 16
        jg  Fill                          ; esp > ebp
        jnz NotZ                          ; we only reset when esp = ebp
        mov edi,_start                    ; reset

  NotZ: mov ecx,15                        ; offset, not counter
        xor eax,eax                       ; clear bits 8-31
        xor edx,edx                       ; zero-out the upper half

  .l0:  mov al,[Buffer+esi+ecx]           ; read byte from buffer
        mov bx,[HexTable+eax*2]           ; get corresponding symbols
        mov [Template+ecx*2+ecx],bx       ; write to template
        mov dl,[AsciiTable+eax]           ; get ASCII symbol
        mov [AsciiPart+ecx],dl            ; write to template
        sub ecx,1                         ; next iteration
        jns .l0                           ; ecx is an offset not a counter

        ; Print the Template to stdout
  Out:  mov eax,SYS_WRITE
        mov ebx,STDOUT
        mov ecx,Template
        mov edx,TEMPLATELENGTH
        int 80h
        cmp eax,TEMPLATELENGTH
        jne Err

        add esi,16                        ; move index
        jmp edi                           ; go to Read or _start

  Last: sub ebp,esi                       ; make it a count
        jz Exit                           ; if 0, we are done
        sub ebp,1                         ; make it an offset

        mov ecx,15                        ; 16*4=64 (0 ... 15)
  .l0:  mov dword [Template+ecx*4],'    ' ; fill buffer with spaces
        sub ecx,1                         ; next iteration
        jns .l0                           ; ecx is an offset
        mov byte [DivPos],'|'             ; it was replaced by a space
        mov word [Template+64],'  '       ; finish writing spaces

        xor ebx,ebx                       ; clear
  .l1:  mov bl,[Buffer+esi+ebp]           ; get byte
        mov cx,[HexTable+ebx*2]           ; get corresponding hex code
        mov [Template+ebp*2+ebp],cx       ; place hex code
        mov bl,[AsciiTable+ebx]           ; get ascii symbol
        mov [AsciiPart+ebp],bl            ; place it
        sub ebp,1                         ; dec doesn't affect CF
        jns .l1

        mov edi,End                       ; return address
        jmp Out                           ; print last line

  Err:  mov ebx,-1                        ; EXIT_FAILURE
        jmp Exit

        ; Exit the program
  End:  xor ebx,ebx                       ; EXIT_SUCCESS
  Exit: mov eax,SYS_EXIT
        int 80h

Answer 1

I would refrain from using esp as general purpose register. This is limiting you in using call/ret subroutines, and you risk memory corruption caused by interrupt happening in your program context.

I would also refrain of such heavy LUT tables usage, as the program will be more likely limited by the I/O operations speed, so that tiny amount of calculation will quite likely hide in the buffered I/O waits.

Here is my version, avoiding those things mentioned above:

; hexdump.asm
; Description     : Prints stdin to stdout as hex and displays ASCII next to it.
;                   It handles correctly the cases in which a sys_read returns
;                   less than was requested but an EOF was not received, this
;                   allows the user to dump arbitrary text from stdin to a file 
;                   in the following way: ./hexdump >> dump.txt
;
; Build using these commands:
;   nasm -f elf hexdump.asm
;   ld -o hexdump hexdump.o -m elf_i386 -s
;
; Usage:
;   ./hexdump < input_file

STDIN equ 0
STDOUT equ 1
SYS_READ equ 3
SYS_WRITE equ 4
SYS_EXIT equ 1
NEW_LINE equ 0ah
BUFFERSIZE equ 16

section .data
HexLUT  db  '0123456789ABCDEF'

section .bss
ReadBuffer: resb BUFFERSIZE
DataBuffer: resb BUFFERSIZE

DIVIDER_OFFSET EQU 3*BUFFERSIZE
ASCII_OFFSET EQU DIVIDER_OFFSET+2
PRINT_BUFFER_SIZE EQU ASCII_OFFSET + BUFFERSIZE + 1
; = BUFFERSIZE * "%02x ", 2 chars for divider "| ", BUFFERSIZE * char + 1 for NEWLINE

PrintBuffer: resb PRINT_BUFFER_SIZE

section .text

global _start ; make it visible

_start:
        lea     esi,[DataBuffer]
        lea     edi,[DataBuffer+BUFFERSIZE]
Read:
        xor     ebx,ebx                 ; STDIN
        lea     ecx,[ReadBuffer]        ; buffer to read into
        mov     edx,BUFFERSIZE          ; BUFFERSIZE bytes at most
        mov     eax,SYS_READ
        int     80h
        cmp     eax,ebx                 ; check return value with 0
        jle     Finish                  ; < 0 -> error, 0 = no more bytes

CopyReadData:
        ; ecx = source of read data, eax = count
        ; esi = current DataBuffer, edi = end of DataBuffer
        cmp     esi,edi
        ja      .DataBufferIsNotFull
        ; display DataBuffer, when full
        call    DisplayDataBuffer
        lea     esi,[DataBuffer]        ; from the beginning of DataBuffer
.DataBufferIsNotFull:
        ; copy eax bytes from @ecx to @esi (clobbers bl)
        mov     bl,[ecx]
        mov     [esi],bl
        inc     ecx
        inc     esi
        dec     eax
        jnz     CopyReadData
        jmp     Read                    ; read more data

DisplayDataBuffer:
    ; esi = end of DataBuffer data
        push    eax
        push    ebx
        push    ecx
        push    edx
        lea     ebx,[PrintBuffer]
    ; print BUFFERSIZE values as hexa "nn " string followed by space
    ; or three "   " spaces in case of no more data in buffer
        lea     ecx,[DataBuffer]
        mov     edx,BUFFERSIZE
PreparePrintBufferHex:
        cmp     ecx,esi
        jz      .noMoreData
        mov     al,[ecx]
        inc     ecx
        call    Format_AL_AsHexWithSpaceTo_EBX
        jmp     .finishLoop
.noMoreData:
        mov     [ebx],DWORD '    '
        add     ebx,3
.finishLoop:
        dec     edx
        jnz     PreparePrintBufferHex
    ; print divider between hexa and ASCII parts
        mov     [ebx],WORD '| '
        add     ebx,2
    ; print ASCII data (or spaces, when no more data)
        lea     ecx,[DataBuffer]
        mov     edx,BUFFERSIZE
PreparePrintBufferAscii:
        cmp     ecx,esi
        mov     al,' '
        jz      .validAscii     ; no more data
        mov     al,[ecx]
        inc     ecx
        cmp     al,0x7F         ; values 00-1F and 7F-FF are shown as '.'
        jae     .invalidAscii
        cmp     al,' '
        jae     .validAscii
.invalidAscii:
        mov     al,'.'
.validAscii:
        mov     [ebx],al
        inc     ebx
        dec     edx
        jnz     PreparePrintBufferAscii
    ; Add new line character at the end of line
        mov     [ebx],BYTE NEW_LINE
    ; Output PrintBuffer to screen
        mov     eax,SYS_WRITE
        mov     ebx,STDOUT
        lea     ecx,[PrintBuffer]
        mov     edx,PRINT_BUFFER_SIZE
        int     80h
        pop     edx
        pop     ecx
        pop     ebx
        pop     eax
        ret

Format_AL_AsHexWithSpaceTo_EBX:
        push    eax
        shr     eax,4
        call    .EAX_nibble
        pop     eax
        call    .EAX_nibble
        mov     [ebx],BYTE ' '
        inc     ebx
        ret

.EAX_nibble:
        and     eax,0x0F
        mov     al,[HexLUT+eax]
        mov     [ebx],al
        inc     ebx
        ret

Finish:
        call    DisplayDataBuffer       ; display any incomplete line of data
        mov     ebx,eax                 ; error code from SYS_READ is exit code
        mov     eax,SYS_EXIT
        int     80h

Answer 2

Some random thoughts:

• Unused defines: STDIN, EXIT_SUCCESS
• I'd restructure the code above Ok: to do jz Last; jl Err and just let jg fall thru (unless Last is further away than it looks).
• I don't believe it's safe to use esp as a general purpose register like this.
• Why do you xor edx,edx above .l0?
• I find your ebp/esp/edi logic confusing. I've got an alternative, but I'm not on linux, so I can't actually run this (and I HATE posting code I haven't run), but how about something like:

.

; Description     : Prints stdin to stdout as hex and displays ASCII next to it.
;                   It handles correctly the cases in which a sys_read returns
;                   less than was requested but an EOF was not received, this
;                   allows the user to dump arbitrary text from stdin to a file 
;                   in the following way: ./hexdump >> dump.txt
;
; Build using these commands:
;   nasm -f elf hexdump.asm
;   ld -o hexdump hexdump.o -m elf_i386 -s
;
; Usage:
;   ./hexdump << input_file

section .data
STDIN: equ 0
STDOUT: equ 1
STDERR: equ 2
EXIT_SUCCESS: equ 0
SYS_READ: equ 3
SYS_WRITE: equ 4
SYS_EXIT: equ 1
NEW_LINE: equ 0ah
AsciiTable: db '................................ !"',"#$%&'()*+,-./0123456789:;"
            db '<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz'
            db '{|}~...........................................................'
            db '...............................................................'
            db '.......'
align 4
Template: db '00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 '
DivPos:   db '| '
AsciiPart: db '................',NEW_LINE
TEMPLATELENGTH: equ $-Template
align 2
HexTable: db '000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2'
          db '02122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f40'
          db '4142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606'
          db '162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f8081'
          db '82838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a'
          db '2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2'
          db 'c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e'
          db '3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff'

section .bss
BUFFERSIZE: equ 16
Buffer: resb BUFFERSIZE

section .text

global _start ; make it visible

_start:
        mov ebx,STDIN
        lea ecx,[Buffer]
        mov edx,BUFFERSIZE
        mov esi,BUFFERSIZE

Fill:
        mov eax,SYS_READ
        int 80h                           ; call
        cmp eax,0                         ; check return value
        jl Err                            ; < 0 -> error
        jz Last                           ; 0 = EOF

        add ecx, eax                      ; Update where to load data
        sub edx, eax                      ; How many more bytes needed?
        jnz Fill                          ; Keep trying until we get 16 bytes

        ; At this point, we have read exactly 16 bytes.

        call OutputLine                   ; Turn esi bytes in Buffer into
                                          ; Template and print it

        jmp _start                        ; Get next line

; At this point, we have hit EOF.  Output any leftovers
; in the buffer.
Last:
        sub esi, edx                      ; How many did we really read?
        jz End                            ; Did we end on exactly 16 bytes?

        ; Blank out the Template and AsciiPart
        mov bx, 0x2020                    ; '  '
        mov edx,BUFFERSIZE

WriteBlanks:
        dec edx
        mov [Template+edx*2+edx],bx       ; write blank over Template
        mov [AsciiPart+edx],bl            ; write blank over AsciiPart
        jnz WriteBlanks

        call OutputLine                   ; Turn esi bytes in Buffer into
                                          ; Template and print it

; Exit the program
End:    mov ebx,EXIT_SUCCESS              ; EXIT_SUCCESS
Exit:   mov eax,SYS_EXIT
        int 80h

Err:    mov ebx,-1                        ; EXIT_FAILURE
        jmp Exit

; Input: esi: Number of chars to process from Buffer
; Clobbers: eax, ebx, edx, edx, esi
OutputLine:
        xor eax, eax
NextChar:                                 ; Process next char
        dec esi
        mov al,[Buffer+esi]               ; read byte from buffer
        mov bx,[HexTable+eax*2]           ; get corresponding symbols
        mov [Template+esi*2+esi],bx       ; write to template
        mov bl,[AsciiTable+eax]           ; get ASCII symbol
        mov [AsciiPart+esi],bl            ; write to template
        jnz NextChar                      ; ecx is an offset not a counter

        mov eax,SYS_WRITE
        mov ebx,STDOUT
        mov ecx,Template
        mov edx,TEMPLATELENGTH
        int 80h
        cmp eax,TEMPLATELENGTH
        jne Err

        ret

It looks right, but even if it is slightly flawed it should give you some ideas.