Windows XP, 16 bit code, and seg fault in winem87.dll under hyper-v with buggy 16 bit handling

User avatar
antus
Site Admin
Posts: 10111
Joined: Sat Feb 28, 2009 10:34 am
cars: TX Gemini 2L Twincam 8psi
TX Gemini SR20 18psi
Datsun 1200 Ute
Subaru Blitzen '06 EZ30 4th gen, 3.0R Spec B
Subaru WRX 2007

Windows XP, 16 bit code, and seg fault in winem87.dll under hyper-v with buggy 16 bit handling

Post by antus »

I was trying to run some old software under windows xp, under hyper-v (under vmware), on a modern amd cpu, and hyper-v is known to have bugs and not return the segment properly for some co-processor calls. This causes segfaults in win87em.dll (part of windows). I've tried emul87 and winfloat's hide87.com but neither was able to solve it. So I let claude at it. It was able to get in to the vm, find a way to pass files back and fourth and run all the tools to properly identify and debug the problem. The result is attached. Its a patched Windows XP winem87.dll and it goes in c:\windows\system32\ and in c:\windows\system32\dllcache\ (hidden folder). Windows will complain that a system file is modified, but you tell it to leave it and it should be happy from then on. And ideally your crashing 16 bit code will work again in a modern vm environment.

Code: Select all

#!/usr/bin/env python3
"""Build WIN87EM.DLL patch v3 (FCS fix + skip re-exec of unidentified insn + diagnostic log) from the ORIGINAL XP DLL.

author: antus @ pcmhacking.net with Claude 2026-11-08

usage: patch_win87em_v3.py <original win87em.dll> <w87v3_cave.bin> <out file>
"""
import hashlib, struct, sys

ORIG_SHA1 = '4132ddc1065209ec62f059673ee7293bd717b0d6'
SEG2_MINALLOC = 0x600

src, cave_path, out = sys.argv[1:4]
raw = open(src, 'rb').read()
assert hashlib.sha1(raw).hexdigest() == ORIG_SHA1, 'input is not the original XP WIN87EM.DLL'
d = bytearray(raw)
cave = open(cave_path, 'rb').read()
assert cave[-4:] == b'TRLR'
a_fcs, a_h2, a_h3, a_fix1 = struct.unpack('<4H', cave[-12:-4])
cave = cave[:-12]

ne = struct.unpack_from('<H', d, 0x3C)[0]
align = struct.unpack_from('<H', d, ne + 0x32)[0]
segtab = ne + struct.unpack_from('<H', d, ne + 0x22)[0]

def seg(i):
    off, ln, fl, mn = struct.unpack_from('<HHHH', d, segtab + (i - 1) * 8)
    return off << align, ln, fl, mn

foff, ln, fl, mn = seg(1)
assert ln == 10289 and mn == 10289 and fl & 0x100
END = ln                                   # cave org (0x2831)

def patch(off, orig, new):
    assert d[foff + off:foff + off + len(orig)] == orig, hex(off)
    assert len(new) == len(orig)
    d[foff + off:foff + off + len(new)] = new

def rel(frm, to):                          # rel16 for a 3-byte call/jmp at frm
    return struct.pack('<H', (to - (frm + 3)) & 0xFFFF)

# hooks
patch(0x02C6, bytes.fromhex('c55e06'), b'\xE8' + rel(0x02C6, a_fcs))
patch(0x06A4, bytes.fromhex('1e53b38b'), b'\xE9' + rel(0x06A4, a_h2) + b'\x90')
patch(0x061B, bytes.fromhex('f6c440'), b'\xE8' + rel(0x061B, a_h3))

# relocation table: move after the cave, re-point the seg2 chain head at our first fixup
rpos = foff + ln
rcnt = struct.unpack_from('<H', d, rpos)[0]
rtab = bytearray(d[rpos:rpos + 2 + rcnt * 8])
found = 0
for i in range(rcnt):
    p = 2 + i * 8
    atype, rtype, off, a, b = struct.unpack_from('<BBHHH', rtab, p)
    if (rtype & 3) == 0 and a == 2 and off == 0x281B:
        assert not (rtype & 4), 'expected non-additive chain'
        struct.pack_into('<H', rtab, p + 2, a_fix1)
        found += 1
assert found == 1, 'seg2 chain head not found'

new_ln = ln + len(cave)
next_seg = seg(2)[0]
assert foff + new_ln + len(rtab) <= next_seg, f'cave too big: {foff + new_ln + len(rtab) - next_seg} bytes over'
old_end = rpos + len(rtab)
d[foff + ln:foff + new_ln] = cave
d[foff + new_ln:foff + new_ln + len(rtab)] = rtab
for i in range(foff + new_ln + len(rtab), old_end):
    d[i] = 0
struct.pack_into('<H', d, segtab + 2, new_ln)
struct.pack_into('<H', d, segtab + 6, new_ln)

# seg2: keep file length, raise min alloc for the log area
s2 = segtab + 8
s2len, s2min = struct.unpack_from('<H', d, s2 + 2)[0], struct.unpack_from('<H', d, s2 + 6)[0]
assert s2len == 0x180 and s2min == 0x180
struct.pack_into('<H', d, s2 + 6, SEG2_MINALLOC)

open(out, 'wb').write(d)
print(f'cave {len(cave)} bytes at 1:{END:04X}; seg1 {ln} -> {new_ln}; slack left {next_seg - (foff + new_ln + len(rtab))}')
print(f'fcs_fix 1:{a_fcs:04X} h2 1:{a_h2:04X} h3 1:{a_h3:04X} chain head 1:{a_fix1:04X}')
print('out sha1', hashlib.sha1(d).hexdigest())

Code: Select all

; 2026-11-08 Antus @ pcmhacking.net with help from Claude
; Fix windows XP on modern CPU under Microsoft hyper-v with buggy 16 bit instruction handling.
;
; put the patched winem87.dll in system32 and system32\dllcache and ignore the warning about
; modified system files (do not restore them)
;
; WIN87EM.DLL patch v3 = v2 logging + FIX: if the faulting x87 instruction cannot be
; identified (Hyper-V zeroes FIP/FCS/FOO/FOS), do NOT re-execute it in 1:061B (that built
; "FADD [0000:0000]" and #GP'd inside the IRQ13 handler -> endless DPMI fault loop).
; Instead just clear the exception (fnclex), mark it handled (AH|=40h) and let WIN87EM
; signal the application as usual.
; Assemble: nasm -f bin w87v3_cave.asm -o w87v3_cave.bin ; build with patch_win87em_v3.py
;
; Hooks (all in segment 1):
;   1:02C6  lds bx,[bp+6]        -> call fcs_fix   (IRQ13 decode; v1 fix + ring log)
;   1:06A4  push ds/push bx/mov bl,8Bh -> jmp h2_sig + nop (exception signalled to app)
;   1:061B  test ah,40h          -> call h3_rex    (re-execute faulting insn, masked)
;
; Log lives in segment 2 at 0x180.. (seg2 minalloc raised 0x180 -> 0x600).
; Find it in ntvdm memory by the signature "W87L".

        bits 16
        org 0x2831                  ; original end of segment 1

RESUME_NODECODE equ 0x02D4
OLD_CHAIN_HEAD  equ 0x281B          ; first seg2 fixup in the original reloc chain
H2_RET          equ 0x06A8

LOG             equ 0x180
SIG0            equ LOG+0x00        ; 'W8'
SIG1            equ LOG+0x02        ; '7L'
CNT_IRQ         equ LOG+0x04        ; dword: IRQ13 decode entries
CNT_FAIL        equ LOG+0x08        ; dword: no usable selector / no ESC -> opcode 0
CNT_ALT         equ LOG+0x0C        ; dword: FCS bad, used interrupted CS
CNT_SIG         equ LOG+0x10        ; dword: exceptions signalled to app (1:06A4)
CNT_REX         equ LOG+0x14        ; dword: re-execution routine (1:061B)
CNT_INT2        equ LOG+0x18        ; dword: INT 2 entries
RING_IDX        equ LOG+0x1C        ; word
LAST_SIG        equ LOG+0x1E        ; word: AX at last signal
REX_OPC         equ LOG+0x20        ; word: [bp+8] at last re-exec
REX_FOO         equ LOG+0x22        ; word: [bp+0Ah]
REX_FOS         equ LOG+0x24        ; word: [bp+0Ch]
CNT_NOESC       equ LOG+0x26        ; dword: selector ok but no ESC byte found
UNKNOWN         equ LOG+0x2A        ; word: 1 = last IRQ13 instruction unidentified
CNT_SKIP        equ LOG+0x2C        ; dword: re-executions skipped
RING            equ LOG+0x40        ; 32 entries x 32 bytes
LOG_END         equ RING + 32*32    ; 0x5C0
; ring entry: +0 env(14: FCW FSW FTW FIP FCS FOO FOS) +14 IP CS FLAGS
;             +20 outcome(0 FCS ok,1 alt CS,2 no sel,3 no ESC) +22 sel +24 opcode word
;             +26 offset of opcode +28 seq(dword)

; ---------------- H1 -----------------------------------------------------
fcs_fix:
        push es
        push cx
        push si
        push di
fix1:   mov ax, fix2+1              ; seg2 selector (reloc chain)
        mov es, ax
        cld
        cmp word [es:SIG0], 'W8'
        jne .init
        cmp word [es:SIG1], '7L'
        je .ok
.init:  xor ax, ax
        mov di, LOG+4
        mov cx, (LOG_END-LOG-4)/2
        rep stosw
        mov word [es:SIG0], 'W8'
        mov word [es:SIG1], '7L'
.ok:    add word [es:CNT_IRQ], 1
        adc word [es:CNT_IRQ+2], 0
        mov di, [es:RING_IDX]
        inc word [es:RING_IDX]
        and di, 31
        mov cl, 5
        shl di, cl
        add di, RING
        push di
        push ds
        push ss
        pop ds
        mov si, bp
        mov cx, 7
        rep movsw                   ; env
        lea si, [bp+22h]
        mov cx, 3
        rep movsw                   ; IP CS FLAGS of the interrupted code
        pop ds
        pop di
        mov ax, [es:CNT_IRQ]
        mov [es:di+28], ax
        mov ax, [es:CNT_IRQ+2]
        mov [es:di+30], ax
        mov ax, [bp+8]              ; FCS
        call sel_ok
        jz .have0
        mov ax, [bp+24h]            ; interrupted CS
        call sel_ok
        jnz .fail2
        mov word [es:di+20], 1
        add word [es:CNT_ALT], 1
        adc word [es:CNT_ALT+2], 0
        jmp short .have
.have0: mov word [es:di+20], 0
.have:  mov [es:di+22], ax
        mov ds, ax
        mov bx, [bp+6]
        mov cx, 4
.scan:  mov al, [bx]
        add al, 28h
        jc .found
        inc bx
        loop .scan
        mov word [es:di+20], 3
        jmp short .fail
.found: mov ax, [bx]
        mov [es:di+24], ax
        mov [es:di+26], bx
        mov word [es:UNKNOWN], 0
        pop di
        pop si
        pop cx
        pop es
        ret                         ; -> 1:02C9, DS:BX = ESC opcode
.fail2: mov word [es:di+20], 2
.fail:  add word [es:CNT_FAIL], 1
        adc word [es:CNT_FAIL+2], 0
        mov word [es:UNKNOWN], 1
        pop di
        pop si
        pop cx
        pop es
        mov word [bp+8], 0
        add sp, 2
        jmp RESUME_NODECODE

; ZF=1 if selector AX is readable and [bp+6]..[bp+6]+4 is within its limit.
sel_ok:
        verr ax
        jnz .ret
        push cx
        push dx
        lsl cx, ax
        mov dx, [bp+6]
        add dx, 4
        jc .bad
        cmp dx, cx
        ja .bad
        xor dx, dx
        jmp short .out
.bad:   or dx, 1
.out:   pop dx
        pop cx
.ret:   ret

; ---------------- H2: 1:06A4 signal to application ----------------------
h2_sig:
        push ds
        push bx
fix2:   mov bx, fix3+1
        mov ds, bx
        cmp word [SIG0], 'W8'
        jne .x
        add word [CNT_SIG], 1
        adc word [CNT_SIG+2], 0
        mov [LAST_SIG], ax
.x:     mov bl, 8Bh
        jmp H2_RET

; ---------------- H3: 1:061B re-execute routine -------------------------
h3_rex:
        push ds
        push bx
fix3:   mov bx, OLD_CHAIN_HEAD
        mov ds, bx
        cmp word [SIG0], 'W8'
        jne .x
        add word [CNT_REX], 1
        adc word [CNT_REX+2], 0
        mov bx, [bp+8]
        mov [REX_OPC], bx
        mov bx, [bp+0Ah]
        mov [REX_FOO], bx
        mov bx, [bp+0Ch]
        mov [REX_FOS], bx
        cmp word [UNKNOWN], 0
        je .x
        add word [CNT_SKIP], 1
        adc word [CNT_SKIP+2], 0
        pop bx
        pop ds
        add sp, 2                   ; drop return to 1:061E -> return to 061B's caller
        test ah, 40h
        jnz .r
        or ah, 40h                  ; as if re-executed
        fnclex                      ; exception handled; leave operands untouched
.r:     ret
.x:     pop bx
        pop ds
        test ah, 40h
        ret

; ---------------- trailer read (and stripped) by the builder -------------
        dw fcs_fix, h2_sig, h3_rex, fix1+1
        db 'TRLR'

You do not have the required permissions to view the files attached to this post.
Have you read the FAQ? For lots of information and links to significant threads see here: http://pcmhacking.net/forums/viewtopic.php?f=7&t=1396