!"#$%&'(#' ).

*+#,#-$.(/' 0'1"#2#
3*45!’2004

!. "#$$%, &. '()*+$,-.

ageev@mail.cir.ru, ik@oasis.apmath.spbu.ru

/+0 -1$,.2 .*3$45%* )*6-57 8-24.-%-9 ()(6)212)(:;$9) 4245$<7

8)2<$,0:540 )*=+23,7$ -1$,.2, -4,-%*,,7$ ,* *,*+2=$ )$=(+>5*5-%
)*6-57 4245$<7. ?)2 @5-< "2A$*+>,7<" *+#-)25<-< 4325*$540 5-5,
A+0 .-5-)-#- %7%-A7, 4A$+*,,7$ 4245$<-9, 4-#+*4(:540 4 <,$,2$<
@.48$)5-% -1$,;2.-%. B CD!&? 248-+>=(:540 4+$A(:;2$ <$5)2.2
-1$,.2 .*3$45%* )*6-57 4245$<:
! % A-)-E.$ 8-24.*:
1. 8-+,-5* (recall)
2. 5-3,-45> (precision)
3. 4)$A,00 5-3,-45> (average precision)
4. 5-3,-45> ,* ()-%,$ 5 A-.(<$,5-% (precision(5))
5. 5-3,-45> ,* ()-%,$ 10 A-.(<$,5-% (precision(10))
6. R-5-3,-45> (R-precision)
7. 11-5-3$3,79 #)*F2. 8-+,-57/5-3,-452,
2=<$)$,,79 8- <$5-A2.$ TREC (11-point matrix
(TREC))
8. 11-5-3$3,79 #)*F2. 8-+,-57/5-3,-452,
<-A2F212)-%*,,79 %*)2*,5 (11-point matrix
(RIRES))
! % A-)-E.$ .+*442F2.*122:
1. 8-+,-5* (recall)
2. 5-3,-45> (precision)
3. *..()*5,-45> (accuracy)
4. -G26.* (error)
5. F-<$)* (F-measure)
H-+>G*0 3*45> 2= @52I <$5)2. 8-A)-6,- 2=(3$,* 2 -824*,* %
+25$)*5()$ [1, 3, 4, 5], -A,*.-, 2,5$)8)$5*122 5$I 2+2 2,7I -1$,-.
=*3*45(: )*=+23*:540. ?-@5-<( <7 8-A)-6,- -82G$< .*EA(: 2=
@52I <$5)2..
 J,*3*+* <7 A*A2< -824*,2$ <$5)2. -1$,.2 .*3$45%* )*6-57
4245$<7 % 8)2<$,$,22 . -A,-<( =*8)-4( ()(6)2.$), * =*5$< -82G$<
<$5-A2.2 (4)$A,$,20 <$5)2. A+0 8-+(3$,20 2,5$#)*+>,7I
8-.*=*5$+$9 .*3$45%* 8-24.*/.+*442F2.*122.
/+0 <,-#2I <$5)2. 48-),7< %-8)-4-< 0%+0$540 4+(3*9, .-#A*
A+0 A*,,-#- =*8)-4* ,$5 )$+$%*,5,7I A-.(<$,5-%. K*8)2<$),
=,*3$,2$ 8-+,-57 % @5-< 4+(3*$ — ,$-8)$A$+$,,-45> 528* 0/0. B
TREC 5*.2$ =*8)-47 ,$ (3257%*:540 8)2 %7324+$,22 <$5)2. [1]. B
CD!&? 8)2,05- 5*.-$ E$ 4-#+*G$,2$: =*8)-47, A+0 .-5-)7I ,$5
)$+$%*,5,7I A-.(<$,5-%, ,$ )*44<*5)2%*:540 8)2 %7324+$,22
<$5)2..

1. 4'1"#2# (- 0(%&'617-8 9%2:0'(1%7

H-+>G2,45%- <$5)2., 8)2<$,0$<7I % 4-%)$<$,,-9 -1$,.$
5$.45-%-#- 8-24.*, -4,-%7%*:540 ,* -5,-G$,22 )$+$%*,5,-452
(8)2,*A+$E,-452) A-.(<$,5* =*8)-4( ()(6)2.$). D64(EA$,2$ 4*<-#-
8-,0520 )$+$%*,5,-452 %7I-A25 =* )*<.2 A*,,-#- A-.(<$,5*. LA$4>
,$-6I-A2<- +2G> -5<$525>, 35- @5- -5,-G$,2$ 2<$$5 4.-)$$
842I-+-#23$4.(: 8)2)-A( 2 (45*,*%+2%*$540 8)0<7< -8)-4-<
@.48$)5-%--1$,;2.-%. !$5)2.2 A+0 ,$(8-)0A-3$,,-#- <,-E$45%*
A-.(<$,5-% -4,-%*,7 ,* 62,*),-9 .+*442F2.*122 A-.(<$,5-%
«)$+$%*,5$,/,$ )$+$%*,5$,» 8- -5,-G$,2: . %76)*,,-<( =*8)-4(.
/*,,7$ <$5)2.2 -4,-%7%*:540 ,* <*5)21$ .+*442F2.*122:
&" !"#"$%&'&(
!"#"$%&'&(

&%)*"&+ ,-,'".+) a b

&" &%)*"&+ ,-,'".+) c d

;-<$#,- 1 *6(%7(/' 2-1'=%"## 9%2:0'(1%7 %17'1-

6#61'0/
LA$4> * — .-+23$45%- A-.(<$,5-%, ,*9A$,,7I 4245$<-9 2
)$+$%*,5,7I 4 5-3.2 =)$,20 @.48$)5-%; b — .-+23$45%- A-.(<$,5-%,
,*9A$,,7I 4245$<-9, ,- ,$ )$+$%*,5,7I 4 5-3.2 =)$,20 @.48$)5-%; c
— .-+23$45%- )$+$%*,5,7I A-.(<$,5-%, ,$ ,*9A$,,7I 4245$<-9; d
— .-+23$45%- ,$)$+$%*,5,7I A-.(<$,5-%, ,$ ,*9A$,,7I 4245$<-9.

1.1 !%$(%1- (recall)

?-+,-5* (recall) %7324+0$540 .*. -5,-G$,2$ ,*9A$,,7I
)$+$%*,5,7I A-.(<$,5-% . -6;$<( .-+23$45%( )$+$%*,5,7I
A-.(<$,5-%:
a
r"
a#c
?-+,-5* I*)*.5$)2=($5 48-4-6,-45> 4245$<7 ,*I-A25> ,(E,7$
8-+>=-%*5$+: A-.(<$,57, ,- ,$ (3257%*$5 .-+23$45%-
,$)$+$%*,5,7I A-.(<$,5-%, %7A*%*$<7I 8-+>=-%*5$+:. K*8)2<$),
$4+2 8-+,-5* )*%,* 50%, 5- @5- =,*325, 35- 8-+-%2,* )$+$%*,5,7I
A-.(<$,5-% 4245$<-9 ,$ ,*9A$,*.

1.2 ;%>(%61. (precision)

M-3,-45> (precision) %7324+0$540 .*. -5,-G$,2$ ,*9A$,,7I
)$+$%*,5,7I A-.(<$,5-% . -6;$<( .-+23$45%( ,*9A$,,7I
A-.(<$,5-%:
a
p"
a#b
M-3,-45> I*)*.5$)2=($5 48-4-6,-45> 4245$<7 %7A*%*5> % 4824.$
)$=(+>5*5-% 5-+>.- )$+$%*,5,7$ A-.(<$,57. K*8)2<$), $4+2
5-3,-45> )*%,* 50%, 5- @5- =,*325, 35- 4)$A2 ,*9A$,,7I A-.(<$,5-%
8-+-%2,* )$+$%*,5,7I 2 8-+-%2,* – ,$)$+$%*,5,7I.

1.3 )22:"-1(%61. (accuracy)

"..()*5,-45> (accuracy) %7324+0$540 .*. -5,-G$,2$ 8)*%2+>,-
8)2,057I 4245$<-9 )$G$,29 . -6;$<( 324+( )$G$,29. N-)<*+>,-:

Accuracy = (a+d) / (a+b+c+d)

?-4.-+>.( <7 8)$A8-+*#*$<, 35- 4245$<* 8)2,2<*$5 )$G$,2$ -
8)2,*A+$E,-452 . A*,,-9 .*5$#-)22 A+0 .*EA-#- A-.(<$,5*
.-++$.122. M*.2< -6)*=-< =,*<$,*5$+> ,$ =*%2425 -5
)*44<*5)2%*$<-9 .*5$#-)22. ?)2 %7324+$,22 -1$,.2 % .*3$45%$
=,*<$,*5$+0 248-+>=-%*+-4> -6;$$ 324+- A-.(<$,5-%
-1$,2%*%G2I40 I-50 67 A+0 -A,-9 .*5$#-)22 (35- 42+>,- <$,>G$
324+* A-.(<$,5-%, ,- @5- 4.*=7%*$540 +2G> ,* <*4G5*6$).
 D5<$52<, 35- <2.)-(4)$A,$,2$ 2 <*.)-(4)$A,$,2$ A+0 @5-9
-1$,.2 A*:5 -A2,*.-%79 )$=(+>5*5.

1.4 *?#<2- (error)

DG26.* (error) %7324+0$540 .*. -5,-G$,2$ ,$8)*%2+>,- 8)2,057I
4245$<-9 )$G$,29 . -6;$<( 324+( )$G$,29. N-)<*+>,-:

Error = (b+c) / (a+b+c+d)

1.5 F-0'"- (F-measure)

F-<$)* 3*45- 248-+>=($540 .*. $A2,*0 <$5)2.*, -6O$A2,0:;*0
<$5)2.2 8-+,-57 2 5-3,-452 % -A,( <$5)2.(. F-<$)* A+0 A*,,-#-
=*8)-4* ()(6)2.2) %7324+0$540 8- F-)<(+$
2
F"
1 1
#
p r
D5<$52< -4,-%,7$ 4%-945%* <$5)2.2 F:
! 0 $ F $1
! $4+2 r " 0 2+2 p " 0 , 5- F " 0
! $4+2r " p , 5- F " r " p
r# p
! min(r , p) $ F $
2

2. @6"'9('(#' 0(%&'617'((/8 0'1"#2

DA,2< 2= %*E,7I %-8)-4-% 8-45)-$,20 <$5)2. % 5$.45-%-< 8-24.$
(-4-6$,,- % 4+(3*$ .+*442F2.*122 2 F2+>5)*122) 0%+0$540 <$5-A
(4)$A,$,20 )$=(+>5*5-%. P5-5 %-8)-4 =*3*45(: -45*$540 ,$
-4%$;$,,7<, ,$4<-5)0 ,* =,*325$+>,-$ %+20,2$ $#- ,* )$=(+>5*57
-1$,.2.
B 4+(3*$ 8-45)-$,20 (4)$A,$,,-9 8- <,-E$45%( =*A*,29 5-9 2+2
2,-9 <,-E$45%$,,-9 <$5)2.2 <-E,- )*44<-5)$5> A%$
8-4+$A-%*5$+>,-452 A$945%29:
! 4,*3*+* %7324+25> <$5)2.2 8- .*EA-<( =*8)-4(
-5A$+>,- 2 =*5$< 2I (4)$A,25>
! ,*952 -6;$$ .-+23$45%- A-.(<$,5-%, -5,-40;2I40
. .*5$#-)20< 5*6.1 2 (E$ ,* 2I -4,-%$ %7324+25>
24.-<(: <$5)2.(
 ?$)%79 48-4-6 %7324+$,20 8)2,05- ,*=7%*5> <2.)-(4)$A,$,2$<
(microaverage), %5-)-9 – <*.)-(4)$A,$,2$< (macroaverage). ?$)%79
48-4-6 I*)*.5$)$, A+0 -1$,.2 =*A*3 8-24.*, % .-5-)7I %*E$,
)$=(+>5*5 % 4)$A,$< 8- =*8)-4(, ,$=*%242<- -5 <-;,-452 -5%$5* ,*
@5-5 =*8)-4. B5-)-9 E$ 48-4-6 ,*G$+ 6-+>G$$ 8)2<$,$,2$ % -1$,.$
.+*442F2.*122 2 F2+>5)*122, #A$ ,$-6I-A2<- (3257%*5> «)*=<$)7»
=*8)-4-%. B CD!&?’2003 8)2 -1$,.$ )$=(+>5*5-% (3*45,2.-%
A-)-E.2 .+*442F2.*122 8)2<$,0+24> -6* 48-4-6* (4)$A,$,20, * %
A-)-E.$ 8-24.* 5-+>.- <2.)-(4)$A,$,2$.

3 4'1"#2# (- A%6$'9%7-1'$.(%61B8 9%2:0'(1%7

!$5)2.2 A+0 4824.* A-.(<$,5-%, -54-)52)-%*,,-#- 8-
)$+$%*,5,-452, (3257%*:5 ,$ 5-+>.- F*.5 ,*+2320 A-.(<$,5* %
4824.$ ,*9A$,,7I A-.(<$,5-%, ,- 2 $#- 8-+-E$,2$ % @5-< 4824.$.
M*.-9 %2A <$5)2. 2<$$5 4<74+ 5-+>.- % -1$,.$ 4245$< 8-24.*.

3.1 ;%>(%61. (- :"%7(' n 9%2:0'(1%7 (precision(n))

M-3,-45> ,* ()-%,$ n A-.(<$,5-% -8)$A$+0$540 .*. .-+23$45%-
)$+$%*,5,7I A-.(<$,5-% 4)$A2 8$)%7I n %7A*,,7I A-.(<$,5-%,
A$+Q,,-$ ,* n [1].
R4+2 4245$<* %7A*+* 6-+$$ n A-.(<$,5-%, 5- @5* %$+232,* )*%,*
5-3,-452 4245$<7 ,* 8$)%7I n A-.(<$,5*I )$=(+>5*5-% =*8)-4*.
R4+2 4245$<* %7A*+* <$,$$ n A-.(<$,5-%, 5- 5-3,-45> ,* ()-%,$ n
A-.(<$,5-% 6(A$5 =*%$A-<- ,$ %7G$ 5-3,-452 4245$<7.
M-3,-45> ,* ()-%,$ n A-.(<$,5-% I*)*.5$)2=($5 48-4-6,-45>
4245$<7 %7A*%*5> )$+$%*,5,7$ A-.(<$,57 % ,*3*+$ 4824.*
)$=(+>5*5-%. K*8)2<$), $4+2 4245$<* %7A*$5 ,$ 6-+$$ 10 A-.(<$,5-%
,* 8$)%-9 45)*,21$, 5- precision(10) -5)*E*$5 .*3$45%- )$=(+>5*5-%
4245$<7, 8-+(3*$<7I ,* 8$)%-9 45)*,21$.
P5* <$5)2.* 2<$$5 )0A ,$A-45*5.-%. B 3*45,-452, A+0 )*=+23,7I
=*8)-4-% <$5)2.2 precision(n) <-#(5 675> ,$4)*%,2<7. K*8)2<$),
A+0 «2A$*+>,-9» 4245$<7, .-5-)*0 %7A*Q5 5-+>.- )$+$%*,5,7$
A-.(<$,57, precision(100)=0.2 A+0 =*8)-4*, 8- .-5-)-<( 4(;$45%($5
20 )$+$%*,5,7I A-.(<$,5-%, 2 precision(100)=0.3 A+0 =*8)-4*, 8-
.-5-)-<( 4(;$45%($5 30 )$+$%*,5,7I A-.(<$,5-%. K$4<-5)0 ,*
2=%$45,7$ ,$A-45*5.2, 5-3,-45> ,* ()-%,$ 0%+0$540 ,$=*<$,2<-9
<$5)2.-9 4-%)$<$,,7I 4245$< 8-24.* 5*. .*., % 3*45,-452,
8-=%-+0$5 -1$,25> 8-+$=,-45> 8$)%-9 45)*,217 -5%$5* 4245$<7 A+0
8-+>=-%*5$+0.
3.2 R-1%>(%61. (R-precision)
R-5-3,-45> )*%,* 5-3,-452 ,* ()-%,$ n A-.(<$,5-% (8. 0) A+0 n
)*%,-#- .-+23$45%( )$+$%*,5,7I A-.(<$,5-% A+0 A*,,-#- =*8)-4*
[1].
/*,,*0 <$5)2.* 8)2=%*,* =*<$,25> 5-3,-45> ,* ()-%,$ % 5$I
4+(3*0I, .-#A* ,$-6I-A2<- (3$45> 6-+>G(: )*=,21( % .-+23$45%$
)$+$%*,5,7I A-.(<$,5-% )*=,7I =*8)-4-%.

3.3 C"'9(BB 1%>(%61. (average precision)

J)$A,00 5-3,-45> A+0 A*,,-#- =*8)-4* -8)$A$+0$540 4+$A(:;2<
-6)*=-< [1]: 8(45> A+0 A*,,-#- =*8)-4* 2<$$540 k )$+$%*,5,7I
A-.(<$,5-%.
M-3,-45> ,* ()-%,$ i-#- )$+$%*,5,-#- A-.(<$,5* prec_rel(i) )*%,*
precision(pos(i)), $4+2 i-9 )$+$%*,5,79 A-.(<$,5 ,*I-A2540 %
)$=(+>5*5*I =*8)-4* ,* 8-=2122 pos(i). R4+2 i-9 )$+$%*,5,79
A-.(<$,5 ,$ ,*9A$,, 5- prec_rel(i)=0.
J)$A,00 5-3,-45> A+0 A*,,-#- =*8)-4* )*%,* 4)$A,$<( =,*3$,2:
%$+232,7 prec_rel(i) 8- %4$< k )$+$%*,5,7< A-.(<$,5*<:
1 k
AvgPrec " % prec_rel(i)
k i "1
D5<$52< -4,-%,7$ 4%-945%* <$5)2.2 «4)$A,00 5-3,-45>»:
! AvgPrec $ recall
! $4+2 )$+$%*,5,7$ A-.(<$,57 ,*I-A0540 5-+>.- %
,*3*+$ 4824.* )$=(+>5*5-%, 5- AvgPrec " recall
! $4+2 )$+$%*,5,7$ A-.(<$,57 )*%,-<$),-
)*48)$A$+$,7 8- 4824.( )$=(+>5*5-%, 5-
AvgPrec & precision' recall
! .-+23$45%- A-.(<$,5-%, )*,E2)-%*,,7I ,2E$
8-4+$A,$#- )$+$%*,5,-#-, ,$ %+20$5 ,* =,*3$,2$
AvgPrec (-54$.*$540 «I%-45»)
J)$A,00 5-3,-45> 8-=%-+0$5 -1$,2%*5> .*3$45%- )*6-57 4245$<7,
(3257%*0 8)2-)25$5 %74-.- )*,E2)-%*,,7I A-.(<$,5-% 8$)$A
A-.(<$,5*<2, ,*I-A0;2<240 % .-,1$ 4824.*. B -5+232$ -5 <$5)2.
precision(n) 2 R-precision, 4)$A,00 5-3,-45> (3257%*$5 %4$
,*9A$,,7$ A-.(<$,57. B 45*5>$ [2] -5<$3*$540, 35- <$5)2.* AvgPrec
-6+*A*$5 %74-.-9 (45-932%-45>: -5,-425$+>,- %*)2*129 -1$,.2
@.48$)5-% 8)2 %7324+$,22 4)$A,$9 -1$,.2 8- <,-E$45%( =*8)-4-%.
4. 11-1%>'>(/D ="-+#2 A%$(%1//1%>(%61#,
#E0'"'((/D A% 0'1%9#2' TREC (11-point matrix
(TREC))
11-5-3$3,79 #)*F2. 8-+,-57/5-3,-452 -5)*E*$5 2=<$,$,2$
5-3,-452 % =*%242<-452 -5 5)$6-%*,29 . 8-+,-5$ 2 A*$5 6-+$$
8-+,(: 2,F-)<*12:, 3$< $A2,*0 <$5)2.* % %2A$ -A,-9 12F)7 [1,
5]. ?- -42 *641244 ,* #)*F2.$ -5.+*A7%*:540 =,*3$,20 8-+,-57, 8-
-42 -)A2,*5 – =,*3$,2$ 5-3,-452 8)2 (4+-%22, 35- )*44<*5)2%*$540
,*3*+>,79 -5)$=-. )$=(+>5*5-% =*8)-4*, ,* .-5-)-< A-452#*$540
=*A*,,79 ()-%$,> 8-+,-57.
/+0 =*8)-4* A+0 .-5-)-#- 2=%$45,- n )$+$%*,5,7I A-.(<$,5-%,
8-+,-5* <-E$5 8)2,2<*5> A24.)$5,7$ =,*3$,20 0, 1/n, 2/n, ... , 1. /+0
5-#-, 35-67 <-E,- 67+- 8-+(3*5> $A2,79 #)*F2. 8-+,-57/5-3,-452
A+0 <,-E$45%* =*8)-4-%
1. )*44<*5)2%*:540 F2.42)-%*,,7$ =,*3$,20 8-+,-57 0.0,
0.1, 0.2, ... , 1.0 (%4$#- 11 =,*3$,29);
2. 248-+>=($540 48$12*+>,*0 8)-1$A()* 2,5$)8-+0122
5-3,-452 A+0 A*,,7I F2.42)-%*,,7I =,*3$,29 8-+,-57;
3. A+0 <,-E$45%* =*8)-4-% 8)-2=%-A2540 (4)$A,$,2$
5-3,-452 A+0 =*A*,,7I ()-%,$9 8-+,-57.
&,5$)8-+2)-%*,,-$ =,*3$,2$ 5-3,-452 )*%,- <*.42<*+>,-<(
=,*3$,2: 5-3,-452 8)2 ()-%,$ 8-+,-57 6-+>G$< 2+2 )*%,7<
=*A*,,-<(.
J,*3*+* <7 -82G$< 8)-1$A()( %7324+$,20 2,5$)8-+2)-%*,,7I
=,*3$,29 5-3,-452 A+0 A*,,-#- =*8)-4* ,* 8)2<$)$, * =*5$< A*A2<
F-)<*+>,-$ -824*,2$ @5-9 8)-1$A()7.
?)2<$) (2= [5], 4<. )24. 1): 8(45> .-++$.120 A-.(<$,5-%
4-A$)E25 20 A-.(<$,5-%, 4 2= .-5-)7I )$+$%*,5,7 =*8)-4(. ?(45>
4245$<* %7A*$5 % .*3$45%$ )$=(+>5*5-% =*8)-4* %4$ @52 A-.(<$,57,
)*,E2)-%*,,7$ 5*., 35- )$+$%*,5,7<2 0%+0:540 8$)%79, %5-)-9,
3$5%$)579 2 805,*A1*579. /+0 )*=+23,7I 4)$=-% )$=(+>5*5-%
8-+,-5* 8)2,2<*$5 =,*3$,20 0.25, 0.5, 0.75 2 1.0. B 4--5%$545%22 4
8)*%2+-< 2,5$)8-+0122, A+0 =,*3$,29 8-+,-57 -5 0 A- 0.5 5-3,-45>
)*%,* 1.0, A+0 =,*3$,29 8-+,-57 0.6 2 0.7 5-3,-45> )*%,* 0.75, A+0
=,*3$,29 8-+,-57 0.8, 0.9 2 1.0 5-3,-45> )*%,* 0.27 (4/15).
 3#6. 2 F"#7-B A%$(%1//1%>(%61# 9$B ('2%1%"%=%
E-A"%6-. ;%>2-0# %<%E(->'(/ E(->'(#B A%$(%1//1%>(%61# 9$B
+#26#"%7-((/8 6"'E%7. !:(21#"(%D $#(#'D –
#(1'"A%$#"%7-((/' E(->'(#B.

D82G$< 8)-1$A()( 8-45)-$,20 11-5-3$3,-#- #)*F2.* 6-+$$

8-A)-6,-. /+0 .*EA-#- =,*3$,20 8-+,-57
ri * (0.0, 0.1, 0.2,! ,1.0) A+0 .*EA-#- =*8)-4* q j %7324+0$540
=,*3$,2$ 2,5$)8-+2)-%*,,-9 5-3,-452 p(ri , q j ) 4+$A(:;2<
-6)*=-<:
! $4+2 ri + recall(q j ) , 5- p(ri , q j ) " 0
! $4+2 ri $ recall(q j ) , 5-
o pos(ri , q j ) )*%,- <2,2<*+>,-9 A+2,$ 4824.*
)$=(+>5*5-% A+0 =*8)-4* q j , ,* .-5-)-9
A-452#*$540 8-+,-5* ri
 o p (ri , q j ) " max
n . pos( ri , q j )
, precision(n) -
(<*.42<*+>,*0 5-3,-45> ,* ,*3*+>,-< -5)$=.$
)$=(+>5*5-% A+2,7 pos(ri , q j ) 2+2 6-+$$).

L,*3$,2$ 5-3,-452 A+0 <,-E$45%* =*8)-4-% (q ) A+0

j

F2.42)-%*,,-#- ()-%,0 8-+,-57 ri )*%,- 4)$A,$<( =,*3$,2:

2,5$)8-+2)-%*,,-9 5-3,-452 A+0 A*,,-#- ()-%,0 8-+,-57:
N
1
Prec(ri ) "
N
% p (r , q )
j "1
i j

G#1'"-1:"-
[1] Program to evaluate TREC results using SMART evaluation
procedures. Documentation. http://www-
nlpir.nist.gov/projects/trecvid/trecvid.tools/trec_eval/README
[2] Buckley C., Voorhees E. Evaluating evaluation measure stability. In
Proc. of the SIGIR'00, pp. 33-40, 2000.
[3] C. J. van Rijsbergen. Information Retrieval. Butterworth's and Co.,
London, U.K., 2 edition, 1979.
[4] &. '()*+$,-., &. K$.)$45>0,-%. D1$,.* 4245$< 5$.45-%-#-
8-24.*. ?)-#)*<<2)-%*,2$.28(4):226-242, 2002.
http://ir.apmath.spbu.ru/ru/papers.html
[5] The Twelfth Text Retrieval Conference (TREC 2003). Appendix 1.
Common Evaluation Measures.
http://trec.nist.gov/pubs/trec12/appendices/measures.ps