使用 qemu 來做 source code level debug linux kernel - 從 decompress 開始

逍遙山水憶秋年

一般對 linux 使用 qemu + gdb 的除錯方式, 大概都是從 start_kernel 開始, 要是想要從 decompress 開始追起的話, 該怎麼辦呢?

我突然對這有了興趣, 看看怎麼處理。

不過要先來編譯 kernel:

make i386_defconfig
make

首先 decompress 這部份的程式碼在 linux-5.1.15/arch/x86/boot/compressed, 編譯好之後, linux-5.1.15/arch/x86/boot/compressed 也有個 vmlinux, 很可惜, 這個不能拿來當成 debug symbol, 所以幫 linux-5.1.15/arch/x86/boot/compressed/Makefile 加上 -g

KBUILD_CFLAGS := -m$(BITS) -O2 
+KBUILD_CFLAGS := -m$(BITS) -O0 -g

一般我還會把 -O2 改成 -O0, 不知道為什麼, -O2 編譯之後, linux-5.1.15/arch/x86/boot/compressed/vmlinux 會有些 symbol 找不到, 例如: parse_elf, handle_relocations, nm linux-5.1.15/arch/x86/boot/compressed/vmlinux 是看不到他們的。

這樣重新編譯之後, linux-5.1.15/arch/x86/boot/compressed/vmlinux 才能拿來當 debug symbol 並拿來載入 gdb。

使用 qemu 執行 kernel

qemu-system-i386 -kernel arch/x86/boot/bzImage 

啟用 gdb:

qemu-system-i386 -kernel arch/x86/boot/bzImage -S -s

我不知道神奇的 -kernel 是怎麼載入 kernel (似乎是 multiboot 或是特別認出 linux kernel), 不過在我的平台, 可以用 b *0x100000 將 kernel 停在 linux-5.1.15/arch/x86/boot/compressed/head_32.S L64, 你一定很疑惑我是怎麼找到這個位址的吧!

qemu 沒有 bochs 的 magic break point, 所以沒辦法很輕鬆的停在某行程式碼, 我用猜的, 然後在 L65 的地方加入無窮迴圈, 中斷 gdb, 使用 i r eip 查看目前的位址, 很幸運, 就這樣被我找到了。

我不確定這就是 linux kernel 的第一行程式碼進入點, 因為還看到一些其他的程式碼, 但應該是了。

linux-5.1.15/arch/x86/boot/compressed/head_32.S

  1 /* SPDX-License-Identifier: GPL-2.0 */
  2 /*
  3  *  linux/boot/head.S
  4  *
  5  *  Copyright (C) 1991, 1992, 1993  Linus Torvalds
  6  */
  7 
  8 /*
  9  *  head.S contains the 32-bit startup code.
 10  *
 11  * NOTE!!! Startup happens at absolute address 0x00001000, which is also where
 12  * the page directory will exist. The startup code will be overwritten by
 13  * the page directory. [According to comments etc elsewhere on a compressed
 14  * kernel it will end up at 0x1000 + 1Mb I hope so as I assume this. - AC]
 15  *
 16  * Page 0 is deliberately kept safe, since System Management Mode code in
 17  * laptops may need to access the BIOS data stored there.  This is also
 18  * useful for future device drivers that either access the BIOS via VM86
 19  * mode.
 20  */
 21 
 22 /*
 23  * High loaded stuff by Hans Lermen & Werner Almesberger, Feb. 1996
 24  */
 25  .text
 26 
 27 #include <linux/init.h>
 28 #include <linux/linkage.h>
 29 #include <asm/segment.h>
 30 #include <asm/page_types.h>
 31 #include <asm/boot.h>
 32 #include <asm/asm-offsets.h>
 33 #include <asm/bootparam.h>
 34 
 35 /*
 36  * The 32-bit x86 assembler in binutils 2.26 will generate R_386_GOT32X
 37  * relocation to get the symbol address in PIC.  When the compressed x86
 38  * kernel isn't built as PIC, the linker optimizes R_386_GOT32X
 39  * relocations to their fixed symbol addresses.  However, when the
 40  * compressed x86 kernel is loaded at a different address, it leads
 41  * to the following load failure:
 42  *
 43  *   Failed to allocate space for phdrs
 44  *
 45  * during the decompression stage.
 46  *
 47  * If the compressed x86 kernel is relocatable at run-time, it should be
 48  * compiled with -fPIE, instead of -fPIC, if possible and should be built as
 49  * Position Independent Executable (PIE) so that linker won't optimize
 50  * R_386_GOT32X relocation to its fixed symbol address.  Older
 51  * linkers generate R_386_32 relocations against locally defined symbols,
 52  * _bss, _ebss, _got and _egot, in PIE.  It isn't wrong, just less
 53  * optimal than R_386_RELATIVE.  But the x86 kernel fails to properly handle
 54  * R_386_32 relocations when relocating the kernel.  To generate
 55  * R_386_RELATIVE relocations, we mark _bss, _ebss, _got and _egot as
 56  * hidden:
 57  */
 58  .hidden _bss
 59  .hidden _ebss
 60  .hidden _got
 61  .hidden _egot
 62 
 63  __HEAD
 64 ENTRY(startup_32)
 65 /* l5: jmp l5 */
 66  cld
 67  /*
 68   * Test KEEP_SEGMENTS flag to see if the bootloader is asking
 69   * us to not reload segments
 70   */
 71  testb $KEEP_SEGMENTS, BP_loadflags(%esi)
 72  jnz 1f
 73 
 74  cli
 75  movl $__BOOT_DS, %eax
 76  movl %eax, %ds
 77  movl %eax, %es
 78  movl %eax, %fs
 79  movl %eax, %gs
 80  movl %eax, %ss
 81 1:
 82 
 83 /*
 84  * Calculate the delta between where we were compiled to run
 85  * at and where we were actually loaded at.  This can only be done
 86  * with a short local call on x86.  Nothing  else will tell us what
 87  * address we are running at.  The reserved chunk of the real-mode
 88  * data at 0x1e4 (defined as a scratch field) are used as the stack
 89  * for this calculation. Only 4 bytes are needed.
 90  */
 91  leal (BP_scratch+4)(%esi), %esp
 92  call 1f
 93 1: popl %ebp
 94  subl $1b, %ebp
 95 
 96 /*
 97  * %ebp contains the address we are loaded at by the boot loader and %ebx
 98  * contains the address where we should move the kernel image temporarily
 99  * for safe in-place decompression.
100  */
101 
102 #ifdef CONFIG_RELOCATABLE
103  movl %ebp, %ebx
104  movl BP_kernel_alignment(%esi), %eax
105  decl %eax
106  addl    %eax, %ebx
107  notl %eax
108  andl    %eax, %ebx
109  cmpl $LOAD_PHYSICAL_ADDR, %ebx
110  jge 1f
111 #endif
112  movl $LOAD_PHYSICAL_ADDR, %ebx
113 1:
114 
115  /* Target address to relocate to for decompression */
116  movl    BP_init_size(%esi), %eax
117  subl    $_end, %eax
118  addl    %eax, %ebx
119 
120  /* Set up the stack */
121  leal boot_stack_end(%ebx), %esp
122 
123  /* Zero EFLAGS */
124  pushl $0
125  popfl
126 
127 /*
128  * Copy the compressed kernel to the end of our buffer
129  * where decompression in place becomes safe.
130  */
131  pushl %esi
132  leal (_bss-4)(%ebp), %esi
133  leal (_bss-4)(%ebx), %edi
134  movl $(_bss - startup_32), %ecx
135  shrl $2, %ecx
136  std
137  rep movsl
138  cld
139  popl %esi
140 
141 /*
142  * Jump to the relocated address.
143  */
144  leal relocated(%ebx), %eax
145  jmp *%eax
146 ENDPROC(startup_32)
147 
148 #ifdef CONFIG_EFI_STUB
149 /*
150  * We don't need the return address, so set up the stack so efi_main() can find
151  * its arguments.
152  */
153 ENTRY(efi_pe_entry)
154 loop:   jmp loop
155  add $0x4, %esp
156 
157  call 1f
158 1: popl %esi
159  subl $1b, %esi
160 
161  popl %ecx
162  movl %ecx, efi32_config(%esi) /* Handle */
163  popl %ecx
164  movl %ecx, efi32_config+8(%esi) /* EFI System table pointer */
165 
166  /* Relocate efi_config->call() */
167  leal efi32_config(%esi), %eax
168  add %esi, 40(%eax)
169  pushl %eax
170 
171  call make_boot_params
172  cmpl $0, %eax
173  je fail
174  movl %esi, BP_code32_start(%eax)
175  popl %ecx
176  pushl %eax
177  pushl %ecx
178  jmp 2f  /* Skip efi_config initialization */
179 ENDPROC(efi_pe_entry)
180 
181 ENTRY(efi32_stub_entry)
182  add $0x4, %esp
183  popl %ecx
184  popl %edx
185 
186  call 1f
187 1: popl %esi
188  subl $1b, %esi
189 
190  movl %ecx, efi32_config(%esi) /* Handle */
191  movl %edx, efi32_config+8(%esi) /* EFI System table pointer */
192 
193  /* Relocate efi_config->call() */
194  leal efi32_config(%esi), %eax
195  add %esi, 40(%eax)
196  pushl %eax
197 2:
198  call efi_main
199  cmpl $0, %eax
200  movl %eax, %esi
201  jne 2f
202 fail:
203  /* EFI init failed, so hang. */
204  hlt
205  jmp fail
206 2:
207  movl BP_code32_start(%esi), %eax
208  leal startup_32(%eax), %eax
209  jmp *%eax
210 ENDPROC(efi32_stub_entry)
211 #endif
212 
213  .text
214 relocated:
215 
216 /*
217  * Clear BSS (stack is currently empty)
218  */
219  xorl %eax, %eax
220  leal _bss(%ebx), %edi
221  leal _ebss(%ebx), %ecx
222  subl %edi, %ecx
223  shrl $2, %ecx
224  rep stosl
225 
226 /*
227  * Adjust our own GOT
228  */
229  leal _got(%ebx), %edx
230  leal _egot(%ebx), %ecx
231 1:
232  cmpl %ecx, %edx
233  jae 2f
234  addl %ebx, (%edx)
235  addl $4, %edx
236  jmp 1b
237 2:
238 
239 /*
240  * Do the extraction, and jump to the new kernel..
241  */
242     /* push arguments for extract_kernel: */
243  pushl $z_output_len /* decompressed length, end of relocs */
244 
245  movl    BP_init_size(%esi), %eax
246  subl    $_end, %eax
247  movl    %ebx, %ebp
248  subl    %eax, %ebp
249  pushl %ebp  /* output address */
250 
251  pushl $z_input_len /* input_len */
252  leal input_data(%ebx), %eax
253  pushl %eax  /* input_data */
254  leal boot_heap(%ebx), %eax
255  pushl %eax  /* heap area */
256  pushl %esi  /* real mode pointer */
257  call extract_kernel /* returns kernel location in %eax */
258  addl $24, %esp
259 
260 /*
261  * Jump to the extracted kernel.
262  */
263  xorl %ebx, %ebx
264  jmp *%eax
265 
266 #ifdef CONFIG_EFI_STUB
267  .data
268 efi32_config:
269  .fill 5,8,0
270  .long efi_call_phys
271  .long 0
272  .byte 0
273 #endif
274 
275 /*
276  * Stack and heap for uncompression
277  */
278  .bss
279  .balign 4
280 boot_heap:
281  .fill BOOT_HEAP_SIZE, 1, 0
282 boot_stack:
283  .fill BOOT_STACK_SIZE, 1, 0
284 boot_stack_end:

head_32.S L144, L145, L213, L214 是跳到 .text relocated 處, 這裡是載入除錯 symbol 的關鍵。

2:
movl BP_code32_start(%esi), %eax
leal startup_32(%eax), %eax
jmp *%eax # 跳到 L214
ENDPROC(efi32_stub_entry)
#endif

.text
relocated:

想辦法得知 %eax 的內容, 其實就是 runtime 的 relocated 位址。

add-symbol-file vmlinux 0x1fa23c4

0x1fa23c4 是在我 qemu 上的值, 這裡就是 .text 的開始, symbol 載入到這位址, 就可以對
linux-5.1.15/arch/x86/boot/compressed/vmlinux 做 source code debug。

list 1 就是我把中斷點設定在
extract_kernel
parse_elf
handle_relocations

startup_32 會在 0x100000, 我從這裡把

movl $LOAD_PHYSICAL_ADDR, %ebx
1:

/* Target address to relocate to for decompression */
movl BP_init_size(%esi), %eax
subl $_end, %eax
addl %eax, %ebx

%ebx 的值印出來。

b *0x100000
b *0x100026
b *0x100078 # linux-5.1.15/arch/x86/boot/compressed/head_32.S L208

list 1. g.sh

1 b *0x100000
2 #b *0x100078
3 add-symbol-file vmlinux 0x1fa23c4
4 b extract_kernel
5 b parse_elf
6 b handle_relocations
7 target remote localhost:1234

gdb -x g.sh



call extract_kernel 之後, L264 就會跳到解開的 kernel - linux-5.1.15/arch/x86/kernel/head_32.S, 從 startup_32 開始執行下去。

b.sh

1  for obj in arch/x86/boot/compressed/head_32.o arch/x86/boot/compressed/misc.o arch/x86/boot/compressed/string.o arch/x86/boot/compressed/cmdline.o arch/x86/boot/compressed/error.o arch/x86/boot/compressed/piggy.o arch/x86/boot/compressed/cpuflags.o arch/x86/boot/compressed/early_serial_console.o arch/x86/boot/compressed/kaslr.o arch/x86/boot/compressed/acpi.o arch/x86/boot/compressed/eboot.o arch/x86/boot/compressed/efi_stub_32.o; do readelf -S $obj | grep -qF .rel.local && { echo "error: $obj has data relocations!" >&2; exit 1; } || true; done; ld -m elf_i386  -pie  --no-dynamic-linker   -T arch/x86/boot/compressed/vmlinux.lds arch/x86/boot/compressed/head_32.o arch/x86/boot/compressed/misc.o arch/x86/boot/compressed/string.o arch/x86/boot/compressed/cmdline.o arch/x86/boot/compressed/error.o arch/x86/boot/compressed/piggy.o arch/x86/boot/compressed/cpuflags.o arch/x86/boot/compressed/early_serial_console.o arch/x86/boot/compressed/kaslr.o arch/x86/boot/compressed/acpi.o arch/x86/boot/compressed/eboot.o arch/x86/boot/compressed/efi_stub_32.o drivers/firmware/efi/libstub/lib.a -o arch/x86/boot/compressed/vmlinux
2   nm arch/x86/boot/compressed/vmlinux | sed -n -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(startup_32\|startup_64\|efi32_stub_entry\|efi64_stub_entry\|efi_pe_entry\|input_data\|_end\|_ehead\|_text\|z_.*\)$/#define ZO_\2 0x\1/p' > arch/x86/boot/zoffset.h
3   gcc -Wp,-MD,arch/x86/boot/.header.o.d  -nostdinc -isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -I./arch/x86/include -I./arch/x86/include/generated  -I./include -I./arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -D__KERNEL__ -m16 -g -Os -DDISABLE_BRANCH_PROFILING -Wall -Wstrict-prototypes -march=i386 -mregparm=3 -fno-strict-aliasing -fomit-frame-pointer -fno-pic -mno-mmx -mno-sse -ffreestanding -fno-stack-protector -mpreferred-stack-boundary=2 -D_SETUP -D__ASSEMBLY__ -DSVGA_MODE=NORMAL_VGA -I./arch/x86/boot   -c -o arch/x86/boot/header.o arch/x86/boot/header.S
4   ld -m elf_i386   -m elf_i386 -T arch/x86/boot/setup.ld arch/x86/boot/a20.o arch/x86/boot/bioscall.o arch/x86/boot/cmdline.o arch/x86/boot/copy.o arch/x86/boot/cpu.o arch/x86/boot/cpuflags.o arch/x86/boot/cpucheck.o arch/x86/boot/early_serial_console.o arch/x86/boot/edd.o arch/x86/boot/header.o arch/x86/boot/main.o arch/x86/boot/memory.o arch/x86/boot/pm.o arch/x86/boot/pmjump.o arch/x86/boot/printf.o arch/x86/boot/regs.o arch/x86/boot/string.o arch/x86/boot/tty.o arch/x86/boot/video.o arch/x86/boot/video-mode.o arch/x86/boot/version.o arch/x86/boot/video-vga.o arch/x86/boot/video-vesa.o arch/x86/boot/video-bios.o -o arch/x86/boot/setup.elf
5   objcopy  -O binary arch/x86/boot/setup.elf arch/x86/boot/setup.bin
6   objcopy  -O binary -R .note -R .comment -S arch/x86/boot/compressed/vmlinux arch/x86/boot/vmlinux.bin
7   arch/x86/boot/tools/build arch/x86/boot/setup.bin arch/x86/boot/vmlinux.bin arch/x86/boot/zoffset.h arch/x86/boot/bzImage