!"#$$%&'()*+,-&.-*/)01-23&'()*+,-&
!45**1675+,-&
8)90(50:&;<3&<=;>&
Zornitsa Kozareva!
USC/ISI!
Marina del Rey, CA!
[email protected]!
www.isi.edu/~kozareva!
!"#$%&'%(')%&'%*"+,%(')%"#-+%#%.)+/0',1%
Q> How do you get a
perfect sun tan?
Question
Search Engine
2%
!"#$%&'%(')%&'%*"+,%(')%"#-+%#%.)+/0',1%
Q> How do you get a
perfect sun tan?
Question
A> Lie in the sun
Search Engine
Answer
3%
!"+5+%&'%*+%/$#,&%$'&#(%
•  4'&+5,%/+#56"%+,78,+/%95'-8&+%&'6):+,$%
5+$58+-#;%<#68;80+/%
•  =>?% @8-+,% #% &'6):+,$% 6';;+60',% #,&% #% ;8/$% '<%
A+(%*'5&/B%$"+(%5+$)5,%#%5#,A+&%;8/$%'<%5+;+-#,$%
&'6):+,$/%=/,899+$/?%
•  =C?%D'*+-+5%%
–  /+#56"%+,78,+/%&'%,'$%/9+#A%')5%;#,7)#7+%
–  #%&'6):+,$%8/%,'$%#,%#,/*+5%$'%#%.)+/0',%
–  5+;+-#,$%&'6):+,$/%:87"$%,'$%"#-+%$"+%#,/*+5%
–  )/+5%/9+,&/%$''%:)6"%0:+%$'%E,&%$"+%#,/*+5%
F%
G)+/0',%H,/*+58,7%=GH?%
•  H/A%$"+%6':9)$+5%#%.)+/0',%8,%,#$)5#;%
;#,7)#7+%#,&%.)86A;(%5+6+8-+%$"+%+I#6$%#,/*+5%
=#,&%/)J68+,$%6',$+I$%$'%-#;8&#$+%$"+%#,/*+5?%
%%%%%KIL%'()*+,-%%!"+5+%*#/%K8,/$+8,%M'5,1%
%% %%%.-*/)0%%%%%N;:%
%%%%% %%%.-*/)0&?,(-@%%O,+%&#(B%<5':%#:',7%"8/%68$(% % %
%
%
%
%
%
%
%
%
%
%
%
%
%
%
%
%
%
%-8+*/%'<%N;:B%OP'%6"'/+%#%*#$+5% %
%
%%%%%%%%%6';'5%$'%/+,&%$'%H;M+5$%K8,/$+8,%#/%#%
%%%%%%%%%%%%%%%%%5+:+:M5#,6+%'<%K8,/$+8,%Q/%M85$"9;#6+L%
%%%%%%%%%%
%
%
R%
T+'9#5&(%U"#;;+,7+%*"86"%VW4X/%!#$/',%*',%on Feb 16th, 2011
IBM’s!
Watson!
WILLIAM WILKINSON’S “AN ACCOUNT OF THE PRINCIPALITIES
OF WALLACHIA AND MOLDOVIA” INSPIRED THIS AUTHOR’S
MOST FAMOUS NOVEL%
S%
Y%
Z#5#&87:/%<'5%GH%
•  V[CM#/+&%#995'#6"+/%
–  \[KU]%%VW4%!#$/',]%@''7;+%
•  ^,'*;+&7+CM#/+&%#,&%D(M58&%#995'#6"+/%
–  VW4%!#$/',]%H99;+%_858]%!';<5#:%H;9"#]%\5)+%
^,'*;+&7+%K-8%%
`%
V[CM#/+&%a#6$'8&%GH%
•  GNK_\VOb%Z[OUK__Vb@%
–  c+$+6$%.)+/0',%$(9+B%#,/*+5%$(9+B%<'6)/B%5+;#0',/%
–  a'5:);#$+%.)+58+/%$'%/+,&%$'%#%/+#56"%+,78,+%
•  ZH__H@K%[K\[VKdHe%
–  [+$58+-+%5#,A+&%&'6):+,$/%
–  W5+#A%8,$'%/)8$#M;+%9#//#7+/%#,&%5+5#,A%
•  Hb_!K[%Z[OUK__Vb@%
–  KI$5#6$%6#,&8&#$+%#,/*+5/%
–  [#,A%6#,&8&#$+/%%
_8:9;8E+&%H56"8$+6$)5+%
Question
Question Analyser
WEB
Search Engine
Human
Paragraph Sorter
Answer Processor
Answer
G)+/0',%Z5'6+//8,7%
=\"8,7/%$'%+I$5#6$%<5':%$"+%.)+/0',?%
•  H,/*+5%\(9+%c+$+60',%
–  &+68&+%$"+%-5A)@&)-+B:&=9+5/',B%9;#6+?%'<%$"+%#,/*+5&
•  G)+5(%a'5:);#0',%
–  U"''/+%C()0:&D):/,0@*&<'5%$"+%V[%/(/$+:%
•  G)+/0',%\(9+%6;#//8E6#0',%
–  V/%$"8/%#%&+E,80',%.)+/0',B%#%:#$"%.)+/0',B%#%;8/$%
.)+/0',1%
•  a'6)/%c+$+60',%
–  a8,&%$"+%.)+/0',%*'5&/%$"#$%#5+%5+9;#6+&%M(%$"+%
#,/*+5%
•  [+;#0',%KI$5#60',%
–  a8,&%5+;#0',/%M+$*++,%+,00+/%8,%$"+%.)+/0',%
2f%
G)+/0',%Z5'6+//8,7%
\"+(X5+% $"+% $*'% /$#$+/% (')% 6');&% M+% 5++,$+58,7% 8<%
(')X5+%65'//8,7%a;'58&#X/%,'5$"+5,%M'5&+5%
• 
• 
• 
• 
H,/*+5%\(9+g%%N_%/$#$+%
G)+5(g%%$*'%/$#$+/B%M'5&+5B%a;'58&#B%,'5$"%
a'6)/g%$"+%$*'%/$#$+/%
[+;#0',/g%%M'5&+5/=a;'58&#B%1IB%,'5$"?%
Example slide from Chris Manning
22%
4#,(%.)+/0',/%6#,%#;5+#&(%M+%
#,/*+5+&%M(%*+M%/+#56"%
Example slide from Chris Manning
23%
V[CM#/+&%G)+/0',%H,/*+58,7%
Example slide from Chris Manning
2F%
\[KU%V[CM#/+&%GH%_(/$+:/%
•  Where do lobsters like to live?
— on the table
•  Where are zebras most likely found?
— in the dictionary
•  What is an invertebrate?
— Dukakis
Michael Dukakis is a member of the Democratic Party, I have
long suspected that elected officials from the Democratic Party
are some previously unclassified form of invertebrate, a totally
spineless creature capable of great noise but no real movement
or action
2R%
Z#5#&87:/%<'5%GH%
•  EFG95*)@&5HH0,57I)*&
–  \[KU]%%VW4%!#$/',]%@''7;+%
•  ^,'*;+&7+CM#/+&%#,&%D(M58&%#995'#6"+/%
–  VW4%!#$/',]%H99;+%_858]%!';<5#:%H;9"#]%\5)+%
^,'*;+&7+%K-8%%
Example slide from Chris Manning
2S%
^,'*;+&7+CM#/+&%#995'#6"+/%=_858?%
•  W)8;&%#%/+:#,06%5+95+/+,$#0',%'<%$"+%.)+5(%
–  \8:+/B%&#$+/B%;'6#0',/B%+,00+/B%,):+586%.)#,00+/%
•  4#9%<5':%$"8/%/+:#,06/%$'%.)+5(%/$5)6$)5+&%
&#$#%%'5%5+/')56+/%
–  @+'/9#0#;%&#$#M#/+/%
–  O,$';'78+/%=!8A89+&8#%8,<'M'I+/B%cWZ+&8#B%!'5&b+$B%
h#7'?%
–  [+/$#)5#,$%5+-8+*%/')56+/%#,&%5+/+5-#0',%/+5-86+/%
–  _68+,0E6%&#$#M#/+/%
2Y%
D(M58&%#995'#6"+/%=VW4%!#$/',?%
•  W)8;&%#%/"#;;'*%/+:#,06%5+95+/+,$#0',%'<%$"+%
.)+5(%
•  @+,+5#$+%#,/*+5%6#,&8&#$+/%)/8,7%V[%:+$"'&/%
–  H)7:+,$+&%*8$"%',$';'78+/%#,&%/+:8C/$5)6$)5+&%
&#$#%
•  _6'5+%+#6"%6#,&8&#$+%)/8,7%586"+5%A,'*;+&7+%
/')56+/%
–  @+'/9#0#;%&#$#M#/+/%
–  \+:9'5#;%5+#/',8,7%
–  \#I',':86#;%6;#//8E6#0',%
2`%
U'::+568#;%_(/$+:/%
!"+5+%8/%$"+%e')-5+%4)/+):%;'6#$+&1%
V,%Z#58/B%a5#,6+%
!"#$X/%$"+%#MM5+-8#0',%<'5%;8:8$+&%
9#5$,+5/"891%
eLZL%
!"#$%#5+%$"+%,#:+/%'<%O&8,X/%5#-+,/1%
D)78,,%#,&%4),8,,%
!"#$%6)55+,6(%8/%)/+&%8,%U"8,#1%
()#,%
!"#$%A8,&%'<%,)$/%#5+%)/+&%8,%:#5i89#,1% #;:',&/%
!"#$%8,/$5):+,$%&'+/%4#I%['#6"%9;#(1% &5):/%
!"#$%8/%$"+%$+;+9"',+%,):M+5%<'5%
_$#,<'5&%N,8-+5/8$(1%
* Handle mainly factoid questions
YSfC`3FC3Fff%
G)+/0',%U;#//8E6#0',%
•  \"+% 95'6+//% M(% *"86"% #% /(/$+:% #,#;(i+/% #%
.)+/0',%#,&%;#M+;/%$"+%.)+/0',%*8$"%#%78-+,%
#,/*+5%$(9+%
&&'%%j!"'%*#/%$"+%E5/$%Z58:+%48,8/$+5%'<%U#,#&#1k%
&.-*/)0&J:H)g%Z+5/',%
•  V$%8/%#,%8,$+75#;%9#5$%'<%#,(%GH%/(/$+:%
–  ;8:8$/%$"+%5#,7+%'<%&#$#l6'59'5#%$'%M+%/+#56"+&%
–  :#A+/%$"+%GH%95'M;+:%$5#6$#M;+%
\#/A%c+E,80',%
•  @8-+,% #% /+$% '<% .)+/0',/B% M)8;&% #,% #)$':#$+&%
/(/$+:% $"#$% 6#,% ;+#5,% $"+% :#998,7% M+$*++,%
$"+%.)+/0',/%#,&%$"+%#,/*+5%$(9+/%
\#/A%c+E,80',%
•  @8-+,% #% /+$% '<% .)+/0',/B% M)8;&% #,% #)$':#$+&%
/(/$+:% $"#$% 6#,% ;+#5,% $"+% :#998,7% M+$*++,%
$"+%.)+/0',/%#,&%$"+%#,/*+5%$(9+/%
•  !"#$X/%8,%#%G)+/0',1%
–  G)+/0',%\(9+%
–  H,/*+5%\(9+%
–  G)+/0',%a'6)/%
–  G)+/0',%\'986%
G)+/0',%\(9+%
•  K)?%% H,% 8&8':#06% 6#$+7'58i#0',% '<% .)+/0',/%
<'5% $"+% 9)59'/+% '<% &8/0,7)8/"8,7% M+$*++,%
&8m+5+,$% 95'6+//8,7% /$5#$+78+/% #,&l'5% #,/*+5%
<'5:#$/L%
–  aHU\OVcg%%%%%%%%%%%jD'*%<#5%8/%8$%<5':%K#5$"%$'%4#5/1k%
–  eV_\g%%%%%%%%%%%%%%%%%%%je8/$%$"+%,#:+/%'<%6"+*8,7%7):/k%
–  cKaVbV\VObg%%%%%j!"'%8/%d;#&%$"+%V:9#;+51k%
Trivia-like
questions
where short,
factual
answers are
expected!
–  hK_CbOg%%%%%%%%%%%%%jV/%_#&&#:%D)//+8,%#;8-+1k%
–  OZVbVObg%%%%%%%%%%j!"#$%&'%:'/$%H:+586#,/%$"8,A%'<%7),%6',$5';1k%
–  n%
H,/*+5%\(9+%
•  K)?%% \"+% 6;#//% '<% 'Mo+6$% =$(9+% '<% /+,$+,6+?%
/')7"$%M(%$"+%.)+/0',L%
– 
– 
– 
– 
ZK[_Ob% %
ZeHUK% %
cH\K% %
bN4WK[%
%%
%
%
%
%
%
%
%
%
%=<5':%j!"'%nk?%
%=<5':%j!"+5+%nk?%
%=<5':%j!"+,%nk?%
%=<5':%jD'*%:#,(%nk?%
%M)$%#;/'%
–  KpZeHbH\VOb% %
–  4K\DOc% % %
%=<5':%j!"(%nk?%
%=<5':%jD'*%nk?%
G)+/0',%a'6)/%
•  K)?%% \"+% 95'9+5$(% '5% +,0$(% $"#$% 8/% M+8,7%
/')7"$%M(%$"+%.)+/0',L%
–  jV,%*"#$%/$#$+%8/%$"+%@5#,&%U#,(',1k%
–  j!"#$%8/%$"+%9'9);#0',%'<%W);7#58#1k%
–  j!"#$%6';'5%8/%#%9':+75#,#$+1k%
G)+/0',%a'6)/%
•  K)?%% \"+% 95'9+5$(% '5% +,0$(% $"#$% 8/% M+8,7%
/')7"$%M(%$"+%.)+/0',L%
–  jV,%*"#$%*B5B)&8/%$"+%@5#,&%U#,(',1k%
–  j!"#$%8/%$"+%H,H(45+,-&'<%W);7#58#1k%
–  j!"#$%7,4,0&8/%#%9':+75#,#$+1k%
G)+/0',%\'986%
•  K)?%% \"+% 'Mo+6$% =9+5/',B% 9;#6+B% n?% % '5% +-+,$%
$"#$% $"+% .)+/0',% 8/% #M')$L% b'$+% $"#$% $"+%
.)+/0',% :87"$% *+;;% M+% #M')$% #% 95'9+5$(% '<%
$"+%$'986B%*"86"%*8;;%M+%$"+%.)+/0',%<'6)/L%
–  j!"#$%8/%$"+%"+87"$%'<%4$L%K-+5+/$1k%
•  I)12IB&8/%$"+%<'6)/%
•  LBM&NO)0)*B&8/%$"+%$'986%
c#$#%U5+#0',%
•  !"'%65+#$+/%$"+%H,/*+5%\(9+/1%
•  D'*% #,&% *"(% #5+% 6+5$#8,% H,/*+5% \(9+/%
/+;+6$+&1%
-  % H,/*+5% \(9+/% #5+% )/)#;;(% 0+&% $'% $"+% 6;#//+/%
5+6'7,8i+&% M(% ')$9)$% '<% b#:+&% K,0$(% [+6'7,80',%
_(/$+:/%
- %%H,/*+5%\(9+/%&+9+,&/%',%$"+%E,#;%#99;86#0',%
\[KU%GH%c#$#%_+$/%
•  K#6"%(+#5%#%,+*%/+$%'<%.)+/0',/%8/%65+#$+&%
•  \"+% .)+/0',/% #5+% ,'$% ;#M+;+&% *8$"% +I9+6$+&%
#,/*+5%$(9+/%
•  \"+%,#:+&%+,0$(%$#7/%#5+%7+,+5#$+&%M(%WWbX/%
V&+,0E,&+5%,#:+&%+,0$(%$#77+5%
•  \"+%5+9'/8$'5(%6',$#8,/%F2%H,/*+5%\(9+/%
KI#:9;+/%'<%H,/*+5%\(9+/%
c8/$58M)0',%'<%H,/*+5%\(9+/%
sparse data
date, definition, GPE, person ~57%
facility description, time are seen only 1
H,/*+5%\(9+%\#I',':(%
p8,%e8B%c#,%['$"L%3ff3L%e+#5,8,7%G)+/0',%U;#//8E+5/L%UOeVb@qf3%
•  Y%6'#5/+%6;#//+/%
–  HWWKdVH\VObB%Kb\V\hB%cK_U[VZ\VObB%DN4HbB%eOUH\VObB%
bN4K[VU%
•  Sf%E,+5%6;#//+/%
–  eOUH\VObg%68$(B%6'),$5(B%:'),$#8,n%
–  DN4Hbg%75')9B%8,&8-8&)#;B%0$;+B%&+/65890',%
–  Kb\V\hg%#,8:#;B%M'&(B%6';'5B%6)55+,6(n%
Example slide from Chris Manning
F2%
e8%r%['$"X/%H,/*+5%\(9+%\#I',':(%
Example slide from Chris Manning
F3%
H,/*+5%\(9+/%8,%T+'9#5&(%
a+55)668%+$%#;L%3f2fL%W)8;&8,7%!#$/',g%H,%O-+5-8+*%'<%$"+%c++9GH%Z5'o+6$L%HV%4#7#i8,+L%a#;;%3f2fL%SuC`uL%
•  3Sff%#,/*+5%$(9+/%8,%3fBfff%T+'9#5&(%
.)+/0',%/#:9;+%
•  \"+%:'/$%<5+.)+,$%3ff%#,/*+5%$(9+/%6'-+5%s%
Sft%'<%&#$#%
•  \"+%Rf%:'/$%<5+.)+,$%T+'9#5&(%#,/*+5%$(9+/%
"+B%6'),$5(B%68$(B%:#,B%E;:B%/$#$+B%/"+B%#)$"'5B%75')9B%"+5+B%6':9#,(B%
95+/8&+,$B%6#98$#;B%/$#5B%,'-+;B%6"#5#6$+5B%*':#,B%58-+5B%8/;#,&B%A8,7B%/',7B%
9#5$B%/+58+/B%/9'5$B%/8,7+5B%#6$'5B%9;#(B%$+#:B%%/"'*B%#6$5+//B%#,8:#;B%
95+/8&+,0#;B%6':9'/+5B%:)/86#;B%,#0',B%M''AB%0$;+B%;+#&+5B%7#:+%
FF%
bC75#:%a+#$)5+/%
•  bC75#:/%#5+%'5&+5+&%#55#,7+:+,$%'<%!"*'5&/%
–  !#$"),875#:%
–  ,v3%M875#:%
–  n%
KIL%jD'*%<#5%8/%8$%<5':%K#5$"%$'%4#5/1k%
% % %),875#:/g%w"'*B%<#5B%8/B%8$x%
% % %M875#:/g%w"'*%<#5B%<#5%8/B%8/%8$%nx%
Is it good to use higher order n-grams? Why?!
FR%
G)+/0',%!'5&%a+#$)5+%
•  N/+%$"+%.)+/0',%*'5&%8$/+;<%#/%#%/+9#5#$+%<+#$)5+%
KIL%jD'*%<#5%8/%8$%<5':%K#5$"%$'%4#5/1k%
% % %),875#:/g%w"'*B%<#5B%8/B%8$x%
%%%%%%%%%%%%%%.)+/0',%*'5&g%w"'*x%
But isn’t this very repetitive? Would it actually help or make a difference?
% % %%
KIL%j!"#$%6';'5%&'+/%;8$:)/%9#9+5%$)5,%*"+,%8$%6':+/%8,$'%6',$#6$%
*8$"%#%/$5',7%#68&1k%
avoid misclassifying
the question as time
FS%
e8:8$#0',/%'<%$"+%c8/6)//+&%a+#$)5+/%
•  \"+(%*8;;%8,6'55+6$;(%6;#//8<(%#%.)+/0',%$"#$%
"#/%:);09;+%$(9+/%
%KIL%!"#$%.)+/0',%
Type
–  !"#$%8/%$"+%95'9+5%,#:+%<'5%#%<+:#;+%%&'()*1%% animal
–  !"#$%8/%b86"';#/%U#7+X/%+(,-.**/,!1%%
bio
cardinal
–  !"#$%8/%$"+%+,+)'&0,!"'<%_+#P;+1%%
cause/effect
–  !"#$%1&)*.2"$"+%e(,:')$"%y''&/1%
–  !"#$%8/%$"+%3.'.+4,!."!)56.(%<'5%$"+%N,8-+5/8$(%'<%
contact info
^+,$)6A(1%
–  !"#$%8/%#,'$"+5%,#:+%<'5%!.&(*/743.2!.**"1% disease
FY%
_+:#,06%a+#$)5+/%
•  e+$X/%8,6'59'5#$+%/':+%/+:#,06/%
•  !"+5+%&'+/%/+:#,06%8,<'5:#0',%6':+%<5':1%
is-a
animal
–  !"#$%8/%$"+%95'9+5%,#:+%<'5%#%<+:#;+%%&'()*1%
is-a
Type
animal
number
–  !"#$%8/%$"+%+,+)'&0,!"'<%_+#P;+1%
cardinal
–  !"#$%8/%$"+%$#;;+/$%5,)!3&/!1%%
location
% %
%%
is-a
location
is-a
structure
C%!"#$%8/%$"+%$#;;+/$%6)/'2/!7"8,%T#9#,1%%
facility
F`%
!'5&b+$%a+#$)5+/%
D+#&*'5&%KI$5#60',%Z5'6+&)5+%
a'5%+#6"%.)+/0',/%
works ~90%
–  5),%ZO_C$#77+5%
–  E,&%$"+%E5/$%,'),%9"5#/+%#,&%#//):+%8$%8/%$"+%:#8,%
',+%
–  +I$5#6$%$"+%587"$:'/$%*'5&%$#77+&%#/%,'),%
–  )/+% !'5&b+$% $'% &+$+5:8,+% #;;% "(9+5,(:/% '<% $"+%
"+#&*'5&%
….
unit
artifact
structure
is-a
!"#$%8/%$"+%3&''.*3"6)/'2/!7"8,%T#9#,1%%
Fz%
!+MCM#/+&%a+#$)5+/%
•  a'5%+#6"%.)+/0',%#,&%8$/%"+#&*'5&B%/)M:8$%$'%
$"+%!+M%;+I86'C/(,$#606%9#P+5,/%'<%$"+%<'5:g%%
%%
%
%
%j{%/)6"%#/%4.&2%,(2k%
•  [+$58+-+% #,&% 5+5#,A% #;;% /+:#,06% 6;#//+/%
;+#5,+&%<5':%$"+%!+M%
!"#$%8/%$"+%95'9+5%,#:+%<'5%#%<+:#;+%%&'()*1%
O$"+5%a+#$)5+/%
•  Z#5$C'<C/9++6"%$#7/%
•  b'),%9"5#/+/%
•  G)+/0',%;+,7$"%
•  b#:+&%K,00+/%="+;9/%7+,+5#;8i+%$"+%.)+/0',?%
%%%%%%!"'%8/%8)9.":''/!73,!1%%%%%%%%sC|%!"'%8/%s9+5/',},#:+|1%
%!"'%8/%;4&('.*"</!26.(741%%sC|%!"'%8/%s9+5/',},#:+|1%
%%
%%
Rf%
4#6"8,+%e+#5,8,7%U;#//8E+5%
•  \"+%:'/$%5'M)/$%9+5<'5:#,6+%8/%#6"8+-+&%*8$"%
_)99'5$%d+6$'5%4#6"8,+/%=_d4?%
•  \5#8,8,7g%
–  a'5%+#6"%.)+/0',%*'5&%=*"#$B%"'*B%*"+,B%n?%M)8;&%
&8m+5+,$%_d4%6;#//8E+5/%
•  \+/0,7g%
–  a'5%+#6"%),/++,%.)+/0',B%+I$5#6$%$"+%.)+/0',%*'5&%
–  W)8;&%#%<+#$)5+%-+6$'5%)/8,7%$"+%/#:+%<+#$)5+/%*"86"%*+5+%
)/+&%8,%$5#8,8,7%
–  N/+%$"+%_d4%6;#//8E+5%6'55+/9',&8,7%$'%$"+%.)+/0',%*'5&%
R2%
\5#8,%_+$%a+#$)5+%@+,+5#0',
\[HVb%cH\H
e+#5,8,7%H;7'58$":
\5#8,+&%4#6"8,+
\K_\%
cH\H
\+/$%_+$%a+#$)5+%@+,+5#0',
N,8-+5/8$(BfB2B2B,);;BN,8B2BRFU%%
N_UBfB2B2B2BfB,);;%%
%#,/*+5
)P5AH4)&
Q0)@17B)@&.-*/)0&
J0()&.-*/)0&
2%
eOUH\VOb%
eOUH\VOb%
2%
f%
eOUH\VOb%
O\DK[%
f%
f%
2%
ZK[_Ob%
ZK[_Ob%
f%
f%
2%
2%
O[@HbV~H\VOb%
O[@HbV~H\VOb%
2%
f%
2%
2%
2%
O\DK[%
O[@HbV~H\VOb%
f%
2%
f%
f%
f%
O\DK[%
O\DK[%
!5HM&
1-K17Q)0&
1-K17R02&
1-K17S,7&
TQ&
2%
2%
2%
f%
f%
f%
f%
2%
2%
2%
Precision =
# correct identified NEs
# identified NEs
!
What is another good measure for the evaluation of question classification?!
R3%
K-#;)#0',%4+#/)5+/%
•  4+#,% [+6895'6#;% [#,A% =4[[?% 8/% )/+&% $'%
+-#;)#$+% #% ;8/$% '<% 9'//8M;+% 5+/9',/+/% $'% #%
.)+5(B%'5&+5+&%M(%95'M#M8;8$(%'<%6'55+6$,+//L%
Q
1
1
MRR = "
Q i=1 RAT
'()*+,-&
!
Q – total number of questions to evaluate
RAT – rank at which answer type is found
F)*(4B&,?&Q0)@17B)@&&
.-*/)0&J:H)&
!"#$% 8/% b86"';#/%
U#7+X/%+(,-.**/,!1%%
9+5/',B%M8'B%6',$#6$%8,<'B%&+E,80',%
!"#$%8/%$"+%$#;;+/$%
6)/'2/!7"8,%T#9#,1%%
+,0$(B%;'6#0',B%79+B%<#68;8$(%
!,00)7B&
F5-D& F)71H0,754&
.-*/)0&J:H)&
F5-D&
M8'%
<#68;8$(%
K-#;)#0',%4+#/)5+/%
•  4+#,% [+6895'6#;% [#,A% =4[[?% 8/% )/+&% $'%
+-#;)#$+% #% ;8/$% '<% 9'//8M;+% 5+/9',/+/% $'% #%
.)+5(B%'5&+5+&%M(%95'M#M8;8$(%'<%6'55+6$,+//L%
Q
1
1
MRR = "
Q i=1 RAT
'()*+,-&
!
Q – total number of questions to evaluate
RAT – rank at which answer type is found
F)*(4B&,?&Q0)@17B)@&&
.-*/)0&J:H)&
!"#$% 8/% b86"';#/%
U#7+X/%+(,-.**/,!1%%
9+5/',B%91,B%6',$#6$%8,<'B%&+E,80',%
!"#$%8/%$"+%$#;;+/$%
6)/'2/!7"8,%T#9#,1%%
+,0$(B%;'6#0',B%79+B%?57141B:&
!,00)7B&
F5-D& F)71H0,754&
.-*/)0&J:H)&
F5-D&
M8'%
3%
2l3%
<#68;8$(%
R%
2lR%
MRR=0.375
G)+/0',%U;#//8E6#0',%Z+5<'5:#,6+%
•  W#/+;8,+%:'&+;%
–  M#7C'<C*'5&% <+#$)5+/% =8L+L% $"+% 8,&8-8&)#;% *'5&/%
$"#$%:#A+%)9%$"+%.)+/0',?%
•  OM$#8,+&%5+/);$/%*8$"%&+/658M+&%<+#$)5+/%
F5-D&F&
V5*)41-)&
":-B5P
W91205AX&
")A5-+7&
":-B5PY")A5-+7&
2%
``LSu%
`zLz2%
zfLFu%
z2L3S%%
3%
zYLR3%
zYLzS%
ufLRS%
zzLSz%
F%
zuLS2%
ufLFf%
uFL2f%
uFLF3%
R%
u2L2Y%
u2L`R%
uSLSS%
uRLRf%
S%
u3LYf%
u3Lz3%
u`LFR%
uSLFF%
2f%
uSLFF%
uSLRf%
uSLRf%
u`L3f%
4[[%
LZ$>[&
LzS2`%
Lz`3`%
LZ[>[&WY>X&
K55'5%H,#;(/8/%
•  V,6',/8/$+,6(%8,%$"+%;#M+;+&%&#$#%
Type
%!"'%8/%8)9.":''/!73,!1%
person
%!"'%8/%;4&('.*"</!26.(741%
biography
% %!"#$%&'+/%;=="/$#,&%<'51%
organization
%!"#$%&'+/%>?@?"/$#,&%<'51%
definition
K55'5%H,#;(/8/%
•  H:M87)')/%#,/*+5%$(9+/%
Type
%!"+5+%*#/%e8,6';,%#//#//8,#$+&1%
facility
location
Type
% %!"+5+%*#/%$"+%#6$5+//B%4#58',%c#-8+/B%M)58+&1%
facility
location
K55'5%H,#;(/8/%
•  V,"+5+,$;(%&8J6);$%.)+/0',/%
Type
%!"#$%8/%$"+%,#:+%'<%$"+%e8',%^8,7X/%/',%8,%$"+%%
animal
%%%:'-8+%j\"+%e8',%^8,7k1%
%%%%!"'%&+-+;'9+&%9'$;#$6"1%
nationality
K55'5%H,#;(/8/%
•  ZO_C$#778,7%:8/6;#//8E6#0',/%%
Type
!"#$%N_%@'-+5,:+,$%#7+,6(%5+78/$+5/%
$5#&+:#5A/1%
!"#$l!Z%N_lbbZ%@'-+5,:+,$lbb%&7.!1Albb%%
5+78/$+5/lbb_%$5#&+:#5A/lbb_%1lL%
Possible solution do a pull parse tree of the question!
organization
K55'5%H,#;(/8/%
•  !'5&b+$%6'-+5#7+%#,&%#:M87)8$(%
Type
!"#$%1.(.&'"7'+/%j/,#9B%65#6A;+B%9'9k1%%%
!'5&b+$%_+,/+%2%
6+5+#;B%6+5+#;%75#//%
%%%%%%%v|%75#//%
%%%%%%%%%%%v|%75#:8,+')/%9;#,$B%75#:8,#6+')/%9;#,$%
%%%%%%%%%%%%%%%v|%"+5MB%"+5M#6+')/%9;#,$%
%%%%%%%%%%%%%%%%%%%v|%-#/6);#5%9;#,$%
product
U#,%(')%M)8;&%(')5%'*,%GU%/(/$+:1%
V,%*"#$%68$(%*#/%Z+$+5%M'5,1%%
!"#$%8/%Z+$+5X/%M85$"9;#6+1%%
!"#$%8/%$"+%M85$"9;#6+%'<%Z+$+51%
b#:+%Z+$+5X/%M85$"9;#6+L%
!"#$%8/%$"+%$#;;+/$%:'),$#8,1%
!"#$%8/%$"+%"87"+/$%:'),$#8,%8,%$"+%*'5;&1%
b#:+%$"+%"87"+/$%:'),$#8,L%
!"#$%8/%$"+%,#:+%'<%$"+%$#;;+/$%:'),$#8,1%
S2%