GNOME Bugzilla – Bug 770295
segfault in convert_Y42B_YUY2 under certain memory conditions
Last modified: 2016-08-25 08:33:09 UTC
Our application crashes with the following backtrace. Process 67884 stopped * thread #70: tid = 0x13c59f, 0x000000010dfed0c5, name = 'multiqueue25:src_0', stop reason = EXC_BAD_ACCESS (code=2, address=0x100000778) frame #0: 0x000000010dfed0c5 -> 0x10dfed0c5: movq %xmm1, (%rax) 0x10dfed0c9: leaq 0x8(%rax), %rax 0x10dfed0cd: leaq 0x4(%rdx), %rdx 0x10dfed0d1: leaq 0x2(%rsi), %rsi (lldb) bt * thread #70: tid = 0x13c59f, 0x000000010dfed0c5, name = 'multiqueue25:src_0', stop reason = EXC_BAD_ACCESS (code=2, address=0x100000778) * frame #0: 0x000000010dfed0c5 frame #1: 0x00000001010d0a78 libgstvideo-1.0.0.dylib`convert_Y42B_YUY2(convert=0x00000001280fd800, src=0x0000700004fe3418, dest=0x0000700004fe3180) + 600 at video-converter.c:3331 frame #2: 0x00000001010c94fa libgstvideo-1.0.0.dylib`gst_video_converter_frame(convert=0x00000001280fd800, src=0x0000700004fe3418, dest=0x0000700004fe3180) + 202 at video-converter.c:2376 frame #3: 0x000000010bff14fe libgstvideoconvert.so`gst_video_convert_transform_frame(filter=0x000000010cb36ad0, in_frame=0x0000700004fe3418, out_frame=0x0000700004fe3180) + 174 at gstvideoconvert.c:692 frame #4: 0x00000001010e1012 libgstvideo-1.0.0.dylib`gst_video_filter_transform(trans=0x000000010cb36ad0, inbuf=0x0000000129a21610, outbuf=0x0000000128668480) + 210 at gstvideofilter.c:271 frame #5: 0x00000001011be489 libgstbase-1.0.0.dylib`default_generate_output + 857 frame #6: 0x00000001011c01c7 libgstbase-1.0.0.dylib`gst_base_transform_chain + 551 frame #7: 0x0000000100b7b861 libgstreamer-1.0.0.dylib`gst_pad_chain_data_unchecked + 193 frame #8: 0x0000000100b7c2cc libgstreamer-1.0.0.dylib`gst_pad_push_data + 204 frame #9: 0x0000000100b68914 libgstreamer-1.0.0.dylib`gst_proxy_pad_chain_default + 212 frame #10: 0x0000000100b7b861 libgstreamer-1.0.0.dylib`gst_pad_chain_data_unchecked + 193 frame #11: 0x0000000100b7c2cc libgstreamer-1.0.0.dylib`gst_pad_push_data + 204 frame #12: 0x00000001010ec7ff libgstvideo-1.0.0.dylib`gst_video_decoder_clip_and_push_buf(decoder=0x00000001287f7ba0, buf=0x0000000129a21610) + 6063 at gstvideodecoder.c:3203 frame #13: 0x00000001010ea7e5 libgstvideo-1.0.0.dylib`gst_video_decoder_finish_frame(decoder=0x00000001287f7ba0, frame=0x0000000000000000) + 1509 at gstvideodecoder.c:3096 frame #14: 0x00000001513dabe0 libgstlibav.so`gst_ffmpegviddec_frame + 3072 frame #15: 0x00000001513d8801 libgstlibav.so`gst_ffmpegviddec_handle_frame + 961 frame #16: 0x00000001010edd0e libgstvideo-1.0.0.dylib`gst_video_decoder_decode_frame(decoder=0x00000001287f7ba0, frame=0x000000010aee7170) + 1822 at gstvideodecoder.c:3405 frame #17: 0x00000001010f685e libgstvideo-1.0.0.dylib`gst_video_decoder_chain_forward(decoder=0x00000001287f7ba0, buf=0x000000010aee76c0, at_eos=0) + 398 at gstvideodecoder.c:2190 frame #18: 0x00000001010f8c6d libgstvideo-1.0.0.dylib`gst_video_decoder_chain(pad=0x000000012a53e6f0, parent=0x00000001287f7ba0, buf=0x000000010aee76c0) + 1805 at gstvideodecoder.c:2492 frame #19: 0x0000000100b7b861 libgstreamer-1.0.0.dylib`gst_pad_chain_data_unchecked + 193 frame #20: 0x0000000100b7c2cc libgstreamer-1.0.0.dylib`gst_pad_push_data + 204 frame #21: 0x00000001011c0113 libgstbase-1.0.0.dylib`gst_base_transform_chain + 371 frame #22: 0x0000000100b7b861 libgstreamer-1.0.0.dylib`gst_pad_chain_data_unchecked + 193 frame #23: 0x0000000100b7c2cc libgstreamer-1.0.0.dylib`gst_pad_push_data + 204 frame #24: 0x0000000101197b85 libgstbase-1.0.0.dylib`gst_base_parse_push_frame + 1445 frame #25: 0x00000001011a01d6 libgstbase-1.0.0.dylib`gst_base_parse_chain + 1094 frame #26: 0x0000000100b7b861 libgstreamer-1.0.0.dylib`gst_pad_chain_data_unchecked + 193 frame #27: 0x0000000100b7c2cc libgstreamer-1.0.0.dylib`gst_pad_push_data + 204 frame #28: 0x000000012c79ed84 libgstcoreelements.so`gst_multi_queue_loop + 1700 frame #29: 0x0000000100babd53 libgstreamer-1.0.0.dylib`gst_task_func + 339 frame #30: 0x0000000100d37223 libglib-2.0.0.dylib`g_thread_pool_thread_proxy + 67 frame #31: 0x0000000100d3622a libglib-2.0.0.dylib`g_thread_proxy + 90 frame #32: 0x00007fff9660299d libsystem_pthread.dylib`_pthread_body + 131 frame #33: 0x00007fff9660291a libsystem_pthread.dylib`_pthread_start + 168 frame #34: 0x00007fff96600351 libsystem_pthread.dylib`thread_start + 13 (lldb) p * ((OrcExecutor *) $rdi) (OrcExecutor) $0 = { program = 0x0000000000000000 n = 570 counter1 = 2 counter2 = 71 counter3 = 0 arrays = { [0] = 0x0000000100000778 [1] = 0x0000000129dbd180 [2] = 0x000000012a53f470 [3] = 0x0000000129dbd180 [4] = 0x00000001edb435e0 [5] = 0x00000001edd99a40 [6] = 0x00000001edf751c0 [7] = 0x0000000100c0dc4c [8] = 0x000000012a53f488 [9] = 0x000000012a53f488 [10] = 0x0000000000000000 [11] = 0x0000000000000008 [12] = 0x0000000129dbb380 [13] = 0x0000000103a6a2b0 [14] = 0x0000000100b7fd06 [15] = 0x0000000000000008 [16] = 0x000000400000460e [17] = 0x0000700004fe2de0 [18] = 0x00007fff979cea26 [19] = 0x0000700004fe2df0 [20] = 0x00007fff979cea26 [21] = 0x0000000000000000 [22] = 0x0000000128668480 [23] = 0x0000700004fe2e90 [24] = 0x0000000100b3e025 [25] = 0xffffffffffffffff [26] = 0x0000000000000000 [27] = 0x0000700004fe2e30 [28] = 0x0000000129dbd030 [29] = 0x0000700004fe2d90 [30] = 0x0000000100c8423b [31] = 0x0000000100c0dc4c [32] = 0x000000012a53f470 [33] = 0x000000000000501e [34] = 0x0000000000000003 [35] = 0x0000000129dbd180 [36] = 0x000000012a462030 [37] = 0x0000700004fe2dd0 [38] = 0x0000000100c8435c [39] = 0x0000000100c0dc4c [40] = 0x000000012a53f470 [41] = 0x0000000100c0dc4c [42] = 0x0000000100d576ca [43] = 0x000000012a53f470 [44] = 0x0000000000000000 [45] = 0x0000700004fe2ec0 [46] = 0x0000700004fe3258 [47] = 0x0000700004fe3258 [48] = 0x00000001ffb21000 [49] = 0x00000001ffb21000 [50] = 0x0000000000010002 [51] = 0x0000700004fe2e40 [52] = 0x0000000100b72d4a [53] = 0x000000012a4600a0 [54] = 0x000000400000501e [55] = 0x000000000000460e [56] = 0x00000001ffb21000 [57] = 0x0000000128668480 [58] = 0x0000000000010002 [59] = 0x0000700004fe2e90 [60] = 0x0000000100b72b1d 2001-01-01 00:00:00 UTC [61] = 0x000000010ae71910 [62] = 0x0000000000000002 [63] = 0x000000012a462270 } params = { [0] = 2280 [1] = 0 [2] = 83767896 [3] = 28672 [4] = 1280 [5] = 640 [6] = 640 [7] = 1 [8] = 0 [9] = 0 [10] = 83767056 [11] = 28672 [12] = 3000 [13] = 759 [14] = 83767192 [15] = 28672 [16] = 3 [17] = 0 [18] = 709238832 [19] = 1 [20] = 11793473 [21] = 1 [22] = 83767040 [23] = 28672 [24] = 83767140 [25] = 28672 [26] = 83767120 [27] = 1 [28] = 1 [29] = 65538 [30] = 83767896 [31] = 28672 [32] = 83767140 [33] = 28672 [34] = 213084880 [35] = 1 [36] = 12639308 [37] = 1 [38] = 83767120 [39] = 28672 [40] = -6 [41] = 0 [42] = 83767216 [43] = 28672 [44] = 17711925 [45] = 1 [46] = 83767192 [47] = 28672 [48] = 12 [49] = 5 [50] = 61597232 [51] = 1 [52] = 702254704 [53] = 1 [54] = 132281128 [55] = 1 [56] = 677807232 [57] = 1 [58] = 0 [59] = 0 [60] = 0 [61] = 0 [62] = 1 [63] = 1 } accumulators = ([0] = 0, [1] = 65538, [2] = 83767784, [3] = 28672) } (lldb) di -s 0x10dfed000 -e 0x10dfed368 0x10dfed000: movl 0x248(%rdi), %eax 0x10dfed006: testl %eax, %eax 0x10dfed008: jle 0x10dfed218 0x10dfed00e: movl %eax, 0x24c(%rdi) 0x10dfed014: movl $0x10, %eax 0x10dfed019: subl 0x18(%rdi), %eax 0x10dfed01c: andl $0xf, %eax 0x10dfed01f: sarl $0x2, %eax 0x10dfed022: cmpl %eax, 0x8(%rdi) 0x10dfed025: jle 0x10dfed03f 0x10dfed027: movl %eax, 0xc(%rdi) 0x10dfed02a: movl 0x8(%rdi), %ecx 0x10dfed02d: subl %eax, %ecx 0x10dfed02f: movl %ecx, %eax 0x10dfed031: sarl $0x3, %ecx 0x10dfed034: movl %ecx, 0x10(%rdi) 0x10dfed037: andl $0x7, %eax 0x10dfed03a: movl %eax, 0x14(%rdi) 0x10dfed03d: jmp 0x10dfed050 0x10dfed03f: movl 0x8(%rdi), %eax 0x10dfed042: movl %eax, 0xc(%rdi) 0x10dfed045: movl $0x0, %eax 0x10dfed04a: movl %eax, 0x10(%rdi) 0x10dfed04d: movl %eax, 0x14(%rdi) 0x10dfed050: movq 0x18(%rdi), %rax 0x10dfed054: movq 0x38(%rdi), %rdx 0x10dfed058: movq 0x40(%rdi), %rsi 0x10dfed05c: movq 0x48(%rdi), %r8 0x10dfed060: testl $0x1, 0xc(%rdi) 0x10dfed067: je 0x10dfed09d 0x10dfed069: movzbl (%rsi), %ecx 0x10dfed06c: movd %ecx, %xmm0 0x10dfed070: movzbl (%r8), %ecx 0x10dfed074: movd %ecx, %xmm1 0x10dfed078: punpcklbw %xmm1, %xmm0 ; xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 0x10dfed07c: pxor %xmm1, %xmm1 0x10dfed080: pinsrw $0x0, (%rdx), %xmm1 0x10dfed085: punpcklbw %xmm0, %xmm1 ; xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 0x10dfed089: movd %xmm1, (%rax) 0x10dfed08d: leaq 0x4(%rax), %rax 0x10dfed091: leaq 0x2(%rdx), %rdx 0x10dfed095: leaq 0x1(%rsi), %rsi 0x10dfed099: leaq 0x1(%r8), %r8 0x10dfed09d: testl $0x2, 0xc(%rdi) 0x10dfed0a4: je 0x10dfed0d9 0x10dfed0a6: pxor %xmm0, %xmm0 0x10dfed0aa: pinsrw $0x0, (%rsi), %xmm0 0x10dfed0af: pxor %xmm1, %xmm1 0x10dfed0b3: pinsrw $0x0, (%r8), %xmm1 0x10dfed0b9: punpcklbw %xmm1, %xmm0 ; xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 0x10dfed0bd: movd (%rdx), %xmm1 ; xmm1 = mem[0],zero,zero,zero 0x10dfed0c1: punpcklbw %xmm0, %xmm1 ; xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 0x10dfed0c5: movq %xmm1, (%rax) 0x10dfed0c9: leaq 0x8(%rax), %rax 0x10dfed0cd: leaq 0x4(%rdx), %rdx 0x10dfed0d1: leaq 0x2(%rsi), %rsi 0x10dfed0d5: leaq 0x2(%r8), %r8 0x10dfed0d9: cmpl $0x0, 0x10(%rdi) 0x10dfed0dd: je 0x10dfed13c 0x10dfed0df: movl 0x10(%rdi), %r9d 0x10dfed0e3: nop 0x10dfed0e4: nop 0x10dfed0e5: nop 0x10dfed0e6: nop 0x10dfed0e7: nop 0x10dfed0e8: nop 0x10dfed0e9: nop 0x10dfed0ea: nop 0x10dfed0eb: nop 0x10dfed0ec: nop 0x10dfed0ed: nop 0x10dfed0ee: nop 0x10dfed0ef: nop 0x10dfed0f0: movd (%rsi), %xmm0 ; xmm0 = mem[0],zero,zero,zero 0x10dfed0f4: movd (%r8), %xmm1 ; xmm1 = mem[0],zero,zero,zero 0x10dfed0f9: punpcklbw %xmm1, %xmm0 ; xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 0x10dfed0fd: movq (%rdx), %xmm1 ; xmm1 = mem[0],zero 0x10dfed101: punpcklbw %xmm0, %xmm1 ; xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 0x10dfed105: movdqa %xmm1, (%rax) 0x10dfed109: movd 0x4(%rsi), %xmm0 ; xmm0 = mem[0],zero,zero,zero 0x10dfed10e: movd 0x4(%r8), %xmm1 ; xmm1 = mem[0],zero,zero,zero 0x10dfed114: punpcklbw %xmm1, %xmm0 ; xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 0x10dfed118: movq 0x8(%rdx), %xmm1 ; xmm1 = mem[0],zero 0x10dfed11d: punpcklbw %xmm0, %xmm1 ; xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 0x10dfed121: movdqa %xmm1, 0x10(%rax) 0x10dfed126: leaq 0x20(%rax), %rax 0x10dfed12a: leaq 0x10(%rdx), %rdx 0x10dfed12e: leaq 0x8(%rsi), %rsi 0x10dfed132: leaq 0x8(%r8), %r8 0x10dfed136: addl $-0x1, %r9d 0x10dfed13a: jne 0x10dfed0f0 0x10dfed13c: testl $0x4, 0x14(%rdi) 0x10dfed143: je 0x10dfed16e 0x10dfed145: movd (%rsi), %xmm0 ; xmm0 = mem[0],zero,zero,zero 0x10dfed149: movd (%r8), %xmm1 ; xmm1 = mem[0],zero,zero,zero 0x10dfed14e: punpcklbw %xmm1, %xmm0 ; xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 0x10dfed152: movq (%rdx), %xmm1 ; xmm1 = mem[0],zero 0x10dfed156: punpcklbw %xmm0, %xmm1 ; xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 0x10dfed15a: movdqu %xmm1, (%rax) 0x10dfed15e: leaq 0x10(%rax), %rax 0x10dfed162: leaq 0x8(%rdx), %rdx 0x10dfed166: leaq 0x4(%rsi), %rsi 0x10dfed16a: leaq 0x4(%r8), %r8 0x10dfed16e: testl $0x2, 0x14(%rdi) 0x10dfed175: je 0x10dfed1aa 0x10dfed177: pxor %xmm0, %xmm0 0x10dfed17b: pinsrw $0x0, (%rsi), %xmm0 0x10dfed180: pxor %xmm1, %xmm1 0x10dfed184: pinsrw $0x0, (%r8), %xmm1 0x10dfed18a: punpcklbw %xmm1, %xmm0 ; xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 0x10dfed18e: movd (%rdx), %xmm1 ; xmm1 = mem[0],zero,zero,zero 0x10dfed192: punpcklbw %xmm0, %xmm1 ; xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 0x10dfed196: movq %xmm1, (%rax) 0x10dfed19a: leaq 0x8(%rax), %rax 0x10dfed19e: leaq 0x4(%rdx), %rdx 0x10dfed1a2: leaq 0x2(%rsi), %rsi 0x10dfed1a6: leaq 0x2(%r8), %r8 0x10dfed1aa: testl $0x1, 0x14(%rdi) 0x10dfed1b1: je 0x10dfed1e7 0x10dfed1b3: movzbl (%rsi), %ecx 0x10dfed1b6: movd %ecx, %xmm0 0x10dfed1ba: movzbl (%r8), %ecx 0x10dfed1be: movd %ecx, %xmm1 0x10dfed1c2: punpcklbw %xmm1, %xmm0 ; xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] 0x10dfed1c6: pxor %xmm1, %xmm1 0x10dfed1ca: pinsrw $0x0, (%rdx), %xmm1 0x10dfed1cf: punpcklbw %xmm0, %xmm1 ; xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7] 0x10dfed1d3: movd %xmm1, (%rax) 0x10dfed1d7: leaq 0x4(%rax), %rax 0x10dfed1db: leaq 0x2(%rdx), %rdx 0x10dfed1df: leaq 0x1(%rsi), %rsi 0x10dfed1e3: leaq 0x1(%r8), %r8 0x10dfed1e7: movl 0x218(%rdi), %ecx 0x10dfed1ed: addl %ecx, 0x18(%rdi) 0x10dfed1f0: movl 0x228(%rdi), %ecx 0x10dfed1f6: addl %ecx, 0x38(%rdi) 0x10dfed1f9: movl 0x22c(%rdi), %ecx 0x10dfed1ff: addl %ecx, 0x40(%rdi) 0x10dfed202: movl 0x230(%rdi), %ecx 0x10dfed208: addl %ecx, 0x48(%rdi) 0x10dfed20b: addl $-0x1, 0x24c(%rdi) 0x10dfed212: jne 0x10dfed014 0x10dfed218: retq 0x10dfed219: addb %al, (%rax) 0x10dfed21b: addb %al, (%rax) 0x10dfed21d: addb %al, (%rax) 0x10dfed21f: addb %ah, 0xf(%rsi) 0x10dfed222: outsb (%rsi), %dx 0x10dfed223: xchgl %edi, 0x2(%rax) 0x10dfed226: addb %al, (%rax) 0x10dfed228: punpcklbw %xmm0, %xmm0 ; xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] 0x10dfed22c: pshuflw $0x0, %xmm0, %xmm0 ; xmm0 = xmm0[0,0,0,0,4,5,6,7] 0x10dfed231: pshufd $0x0, %xmm0, %xmm0 ; xmm0 = xmm0[0,0,0,0] 0x10dfed236: movl $0x10, %eax 0x10dfed23b: subl 0x18(%rdi), %eax 0x10dfed23e: andl $0xf, %eax 0x10dfed241: cmpl %eax, 0x8(%rdi) 0x10dfed244: jle 0x10dfed25e 0x10dfed246: movl %eax, 0xc(%rdi) 0x10dfed249: movl 0x8(%rdi), %ecx 0x10dfed24c: subl %eax, %ecx 0x10dfed24e: movl %ecx, %eax 0x10dfed250: sarl $0x5, %ecx 0x10dfed253: movl %ecx, 0x10(%rdi) 0x10dfed256: andl $0x1f, %eax 0x10dfed259: movl %eax, 0x14(%rdi) 0x10dfed25c: jmp 0x10dfed26f 0x10dfed25e: movl 0x8(%rdi), %eax 0x10dfed261: movl %eax, 0xc(%rdi) 0x10dfed264: movl $0x0, %eax 0x10dfed269: movl %eax, 0x10(%rdi) 0x10dfed26c: movl %eax, 0x14(%rdi) 0x10dfed26f: movq 0x18(%rdi), %rax 0x10dfed273: testl $0x1, 0xc(%rdi) 0x10dfed27a: je 0x10dfed28a 0x10dfed27c: movdqa %xmm0, %xmm1 0x10dfed280: movd %xmm1, %ecx 0x10dfed284: movb %cl, (%rax) 0x10dfed286: leaq 0x1(%rax), %rax 0x10dfed28a: testl $0x2, 0xc(%rdi) 0x10dfed291: je 0x10dfed2a1 0x10dfed293: movdqa %xmm0, %xmm1 0x10dfed297: pextrw $0x0, %xmm1, (%rax) 0x10dfed29d: leaq 0x2(%rax), %rax 0x10dfed2a1: testl $0x4, 0xc(%rdi) 0x10dfed2a8: je 0x10dfed2b6 0x10dfed2aa: movdqa %xmm0, %xmm1 0x10dfed2ae: movd %xmm1, (%rax) 0x10dfed2b2: leaq 0x4(%rax), %rax 0x10dfed2b6: testl $0x8, 0xc(%rdi) 0x10dfed2bd: je 0x10dfed2cb 0x10dfed2bf: movdqa %xmm0, %xmm1 0x10dfed2c3: movq %xmm1, (%rax) 0x10dfed2c7: leaq 0x8(%rax), %rax 0x10dfed2cb: cmpl $0x0, 0x10(%rdi) 0x10dfed2cf: je 0x10dfed2fa 0x10dfed2d1: movl 0x10(%rdi), %edx 0x10dfed2d4: nop 0x10dfed2d5: nop 0x10dfed2d6: nop 0x10dfed2d7: nop 0x10dfed2d8: nop 0x10dfed2d9: nop 0x10dfed2da: nop 0x10dfed2db: nop 0x10dfed2dc: nop 0x10dfed2dd: nop 0x10dfed2de: nop 0x10dfed2df: nop 0x10dfed2e0: movdqa %xmm0, %xmm1 0x10dfed2e4: movdqa %xmm1, (%rax) 0x10dfed2e8: movdqa %xmm0, %xmm1 0x10dfed2ec: movdqa %xmm1, 0x10(%rax) 0x10dfed2f1: leaq 0x20(%rax), %rax 0x10dfed2f5: addl $-0x1, %edx 0x10dfed2f8: jne 0x10dfed2e0 0x10dfed2fa: testl $0x10, 0x14(%rdi) 0x10dfed301: je 0x10dfed30f 0x10dfed303: movdqa %xmm0, %xmm1 0x10dfed307: movdqu %xmm1, (%rax) 0x10dfed30b: leaq 0x10(%rax), %rax 0x10dfed30f: testl $0x8, 0x14(%rdi) 0x10dfed316: je 0x10dfed324 0x10dfed318: movdqa %xmm0, %xmm1 0x10dfed31c: movq %xmm1, (%rax) 0x10dfed320: leaq 0x8(%rax), %rax 0x10dfed324: testl $0x4, 0x14(%rdi) 0x10dfed32b: je 0x10dfed339 0x10dfed32d: movdqa %xmm0, %xmm1 0x10dfed331: movd %xmm1, (%rax) 0x10dfed335: leaq 0x4(%rax), %rax 0x10dfed339: testl $0x2, 0x14(%rdi) 0x10dfed340: je 0x10dfed350 0x10dfed342: movdqa %xmm0, %xmm1 0x10dfed346: pextrw $0x0, %xmm1, (%rax) 0x10dfed34c: leaq 0x2(%rax), %rax 0x10dfed350: testl $0x1, 0x14(%rdi) 0x10dfed357: je 0x10dfed367 0x10dfed359: movdqa %xmm0, %xmm1 0x10dfed35d: movd %xmm1, %ecx 0x10dfed361: movb %cl, (%rax) 0x10dfed363: leaq 0x1(%rax), %rax 0x10dfed367: retq (lldb) re r General Purpose Registers: rax = 0x0000000100000778 sprockets`_mh_execute_header + 1912 rbx = 0x00000001ede16f40 rcx = 0x0000000000000047 rdx = 0x00000001edb435e0 rdi = 0x0000700004fe2c50 rsi = 0x00000001edd99a40 rbp = 0x0000700004fe2fd0 rsp = 0x0000700004fe2c28 r8 = 0x00000001edf751c0 r9 = 0x0000000000000000 r10 = 0x000000000000023a r11 = 0x0000000000000280 r12 = 0x0000000000000bb8 r13 = 0x0000000100c0dc4c libgstreamer-1.0.0.dylib`_gst_debug_min r14 = 0x0000700004fe2c50 r15 = 0x0000000000000002 rip = 0x000000010dfed0c5 rflags = 0x0000000000000202 cs = 0x000000000000002b fs = 0x0000000000000000 gs = 0x0000000000000000 (lldb) up frame #1: 0x00000001010d0a78 libgstvideo-1.0.0.dylib`convert_Y42B_YUY2(convert=0x00000001280fd800, src=0x0000700004fe3418, dest=0x0000700004fe3180) + 600 at video-converter.c:3331 3328 d = FRAME_GET_LINE (dest, convert->out_y); 3329 d += (GST_ROUND_UP_2 (convert->out_x) * 2); 3330 -> 3331 video_orc_convert_Y42B_YUY2 (d, 3332 FRAME_GET_STRIDE (dest), sy, 3333 FRAME_GET_Y_STRIDE (src), su, 3334 FRAME_GET_U_STRIDE (src), sv, (lldb) p d (guint8 *) $1 = 0x00000001ffb21090 (lldb) p sy (guint8 *) $2 = 0x00000001ed8870e0 (lldb) p su (guint8 *) $3 = 0x00000001edc3b7c0 (lldb) p sv (guint8 *) $4 = 0x00000001ede16f40
As per discussion in IRC the root cause is in the following code: 0x10dfed1e7: movl 0x218(%rdi), %ecx 0x10dfed1ed: addl %ecx, 0x18(%rdi) 0x10dfed1f0: movl 0x228(%rdi), %ecx 0x10dfed1f6: addl %ecx, 0x38(%rdi) 0x10dfed1f9: movl 0x22c(%rdi), %ecx 0x10dfed1ff: addl %ecx, 0x40(%rdi) 0x10dfed202: movl 0x230(%rdi), %ecx 0x10dfed208: addl %ecx, 0x48(%rdi) addq should be used to increment strides on the pointers in 64 bits systems. See https://cgit.freedesktop.org/gstreamer/orc/tree/orc/orcprogram-sse.c#n565
Created attachment 334051 [details] [review] orc.diff Part of the problem is this. The size was not taken into account. Unfortunately this breaks the testsuite, so it's probably not complete.
My attempt is here: https://cgit.freedesktop.org/~wtay/orc/commit/?h=add-stride- fix&id=84e5f8ff666f9aea9945de6640f3b4a14a423bd7
Solves the problem for Josep and does not break anything for me. Let's merge this? :) I like your solution more than mine
Merged now: https://cgit.freedesktop.org/gstreamer/orc/commit/?id=46bf82776efd559830b916af6dbf0e96cc338c4d