9:41

MaxxBench – "Nice DeepSWE score. But is it true?"

.

. , , . , , , .

. . , .

. 89 , .

: ...

, . . . . . .

, . 2 .

. , .

, . . ; . ..

. , .

, ; .

/ ___.

/ ___.

. .

, . , . .

, 17 (3.5%). .

...

5.5 3.8 74%. , 76.4% 53.4%.

23- . ; .

-6 5 , 71.9% 63.5% .

74.2%

74.1%

72.7%

69.9%

73.6%

69.0%

74.2%

63.4%

73.8%

67.2%

54.9%

53.3%

12 , , 20.9 48.3 . 1.5 74% , 23.0 .

. 5.5 76.4%, 74.2% , -6 -5.6 2.2 3.6 , . 74% , 4.1 3.8 18.6 20.5 .

.

, . , .

64 , 5.3 64, 253. , : “ 64; >2^53 .” 253 + 1, . , , (“ ? ; ”) (“ / , ”).

. , 5 : “ … .” , . 5 “ ,” — “ 50/50 , 0.55 0.80” — , . , 5, 5.3, 5.3 , .

. 5.5 100% , 16.7% , 30.8% 5 40.3% 5.3.

5.5

-6

-5.6

5

5

-5.3

4.1

4.1

-5.3

3.8

1.3

-5.6

1.2

4

-5.3

40.3%(31 77 )

4.1

36.7%(29 79 )

5

30.8%(20 65 )

-5.3

27.5%(22 80 )

-5.6

18.2%(10 55 )

5.5

16.7%(7 42 )

-5.6

14.6%(14 96 )

-6

12.0%(6 50 )

5

10.8%(7 65 )

4

8.6%(11 128 )

1.3

1.2%(1 83 )

3.8

0%(0 83 )

1.2

0%(0 109 )

, ’ .

. 5 , :

“ … … .”

1,000,100 1,000,900 . 0 [0, 1], .

, :

“ ”

. .

__ “0”. .

“ __ "0" . … "0","","" "" . . .”

, , , .

“ ” :

, .() .

, :

, ( := ()): .() .

. , , =<> ___=<>.

___ . .

13 , . 13 , , .

5.5

-6

-5.6

5

14

.

5

11

, .

-5.3

. . 14

.

4.1

. . 70

.

-5.3

. . 13

.

3.8

1.3

34

, .

-5.6

1.2

4

. 105

.

4.1 . _ _. , , , :

; . 130 ↗

. . , 20 178 .

. , “ , .” , “ .”

4.1 , 100 ↗, , . , , , .

, . , : .

, . , . . , , . 33 , . ..

58 ↗

`.()`! " ".

, 107 ↗, 108 ↗

, .

, 110 ↗ , 114 ↗

/.

/.

35 / 68

469 / 469

↗

.

, : , . .

, . , , , .

, ( [1,2]) {} , 39’ :

(39 §7.3.2)

, 89 . 65.8% 64.5%, 1.3 , .

5.5

0%(0 178 )

-6

0%(0 178 )

-5.6

1.1%(2 178 )

5

1.7%(3 178 )

5

1.7%(3 178 )

-5.3

1.1%(2 178 )

4.1

1.7%(3 178 )

-5.3

2.2%(4 178 )

3.8

1.7%(3 178 )

1.3

2.2%(4 178 )

-5.6

2.2%(4 178 )

1.2

0.6%(1 178 )

4

1.1%(2 178 )

. , .

. , .

, , . , , .

. , , , , .

:

( ; )