| [0pt][] Reproduction Test# Fully reproduce 20.5% Fail to cover some cases 55.5% Fail to reproduce the issue 12.5% Do not know how to run the repo 11.5% [0pt][] Regression Test# Run all related tests correctly 52.0% Miss some related tests 17.5% Miss all related tests 17.0% Do not know how to run the repo 13.5% [0pt][] Edit Location File-path precision 69.0% File-path recall 74.9% Line-number precision 43.2% Line-number recall 48.2% [0pt][] Execution Context File-path precision* 38.2% File-path recall* 72.7% Function precision** 26.4% Function recall** 65.1% [0pt][] API Usage API function-name precision 41.8% API function-name recall 41.9% API definition localization accuracy96.5 Accuracy of API function definition localization is calculated among the agent-extracted API functions with correct function names. Appendix I Examples of Extracted Factor I.1 Regression Test ⬇ { "instance_id" : "verified/django__django-10999" , "command" : "python ./tests/run.py --verbosity 2 --settings=test_pgsql --parallel 1 utils_tests.test_dateparse" , "PASS_TO_PASS" : [ "test_parse_date (utils_tests.test_dateparse.DateParseTests)" , "test_parse_datetime (utils_tests.test_dateparse.DateParseTests)" , "test_parse_time (utils_tests.test_dateparse.DateParseTests)" , "test_days (utils_tests.test_dateparse.DurationParseTests)" , "test_fractions_of_seconds (utils_tests.test_dateparse.DurationParseTests)" , "test_hours_minutes_seconds (utils_tests.test_dateparse.DurationParseTests)" , "test_iso_8601 (utils_tests.test_dateparse.DurationParseTests)" , "test_minutes_seconds (utils_tests.test_dateparse.DurationParseTests)" , "test_parse_python_format (utils_tests.test_dateparse.DurationParseTests)" , "test_seconds (utils_tests.test_dateparse.DurationParseTests)" ] } I.2 Reproduction Test Instance_id: verified/django__django-16560.txt Reproduction Test Code: ⬇ =========================== # constraints.tests.BaseConstraintTests.test_custom_violation_code_message.test_custom_violation_code_message: 1| from unittest import mock 2| 3| from django . core . exceptions import ValidationError 4| from django . db import IntegrityError , connection , models 5| from django . db . models import F 6| from django . db . models . constraints import BaseConstraint , UniqueConstraint 7| from django . db . models . functions import Lower 8| from django . db . transaction import atomic 9| from django . test import SimpleTestCase , TestCase , skipIfDBFeature , skipUnlessDBFeature 10| from django . test . utils import ignore_warnings 11| from django . utils . deprecation import RemovedInDjango60Warning 12| 13| from . models import ( 14| ChildModel , 15| ChildUniqueConstraintProduct , 16| Product , 17| UniqueConstraintConditionProduct , 18| UniqueConstraintDeferrable , 19| UniqueConstraintInclude , 20| UniqueConstraintProduct , 21|) 22| ...... 29| class BaseConstraintTests ( SimpleTestCase ): ...... 80| def test_custom_violation_code_message ( self ): 81| c = BaseConstraint ( name = "base_name" , violation_error_code = "custom_code" ) 82| self . assertEqual ( c . violation_error_code , "custom_code" ) =========================== # constraints.tests.CheckConstraintTests.test_eq.test_eq: 1| from unittest import mock 2| 3| from django . core . exceptions import ValidationError 4| from django . db import IntegrityError , connection , models 5| from django . db . models import F 6| from django . db . models . constraints import BaseConstraint , UniqueConstraint 7| from django . db . models . functions import Lower 8| from django . db . transaction import atomic 9| from django . test import SimpleTestCase , TestCase , skipIfDBFeature , skipUnlessDBFeature 10| from django . test . utils import ignore_warnings 11| from django . utils . deprecation import RemovedInDjango60Warning 12| 13| from . models import ( 14| ChildModel , 15| ChildUniqueConstraintProduct , 16| Product , 17| UniqueConstraintConditionProduct , 18| UniqueConstraintDeferrable , 19| UniqueConstraintInclude , 20| UniqueConstraintProduct , 21|) 22| ...... 120| class CheckConstraintTests ( TestCase ): 121| def test_eq ( self ): 122| check1 = models . Q ( price__gt = models . F ( "discounted_price" )) 123| check2 = models . Q ( price__lt = models . F ( "discounted_price" )) 124| self . assertEqual ( 125| models . CheckConstraint ( check = check1 , name = "price" ), 126| models . CheckConstraint ( check = check1 , name = "price" ), 127| ) 128| self . assertEqual ( models . CheckConstraint ( check = check1 , name = "price" ), mock . ANY ) 129| self . assertNotEqual ( 130| models . CheckConstraint ( check = check1 , name = "price" ), 131| models . CheckConstraint ( check = check1 , name = "price2" ), 132| ) 133| self . assertNotEqual ( 134| models . CheckConstraint ( check = check1 , name = "price" ), 135| models . CheckConstraint ( check = check2 , name = "price" ), 136| ) 137| self . assertNotEqual ( models . CheckConstraint ( check = check1 , name = "price" ), 1) 138| self . assertNotEqual ( 139| models . CheckConstraint ( check = check1 , name = "price" ), 140| models . CheckConstraint ( 141| check = check1 , name = "price" , violation_error_message = "custom error" 142| ), 143| ) 144| self . assertNotEqual ( 145| models . CheckConstraint ( 146| check = check1 , name = "price" , violation_error_message = "custom error" 147| ), 148| models . CheckConstraint ( 149| check = check1 , name = "price" , violation_error_message = "other custom error" 150| ), 151| ) 152| self . assertEqual ( 153| models . CheckConstraint ( 154| check = check1 , name = "price" , violation_error_message = "custom error" 155| ), 156| models . CheckConstraint ( 157| check = check1 , name = "price" , violation_error_message = "custom error" 158| ), 159| ) 160| self . assertNotEqual ( 161| models . CheckConstraint ( check = check1 , name = "price" ), 162| models . CheckConstraint ( 163| check = check1 , name = "price" , violation_error_code = "custom_code" 164| ), 165| ) 166| self . assertEqual ( 167| models . CheckConstraint ( 168| check = check1 , name = "price" , violation_error_code = "custom_code" 169| ), 170| models . CheckConstraint ( 171| check = check1 , name = "price" , violation_error_code = "custom_code" 172| ), 173| ) =========================== ’’ ’ <...As the content is too long, parts of this example have been omitted; the actual information provided to the agent is complete...> ’ ’’ =========================== # constraints.tests.UniqueConstraintTests.test_validate_conditon_custom_error.test_validate_conditon_custom_error: 1| from unittest import mock 2| 3| from django . core . exceptions import ValidationError 4| from django . db import IntegrityError , connection , models 5| from django . db . models import F 6| from django . db . models . constraints import BaseConstraint , UniqueConstraint 7| from django . db . models . functions import Lower 8| from django . db . transaction import atomic 9| from django . test import SimpleTestCase , TestCase , skipIfDBFeature , skipUnlessDBFeature 10| from django . test . utils import ignore_warnings 11| from django . utils . deprecation import RemovedInDjango60Warning 12| 13| from . models import ( 14| ChildModel , 15| ChildUniqueConstraintProduct , 16| Product , 17| UniqueConstraintConditionProduct , 18| UniqueConstraintDeferrable , 19| UniqueConstraintInclude , 20| UniqueConstraintProduct , 21|) 22| ...... 336| class UniqueConstraintTests ( TestCase ): ...... 811| @skipUnlessDBFeature ( "supports_partial_indexes" ) 812| def test_validate_conditon_custom_error ( self ): 813| p1 = UniqueConstraintConditionProduct . objects . create ( name = "p1" ) 814| constraint = UniqueConstraintConditionProduct . _meta . constraints [0] 815| constraint . violation_error_message = "Custom message" 816| constraint . violation_error_code = "custom_code" 817| msg = "Custom message" 818| with self . assertRaisesMessage ( ValidationError , msg ) as cm : 819| constraint . validate ( 820| UniqueConstraintConditionProduct , 821| UniqueConstraintConditionProduct ( name = p1 . name , color = None ), 822| ) 823| self . assertEqual ( cm . exception . code , "custom_code" ) Reproduction Test Command: ⬇ ./tests/runtests.py \ --verbosity 2 \ --settings = test_sqlite \ --parallel 1 \ constraints . tests . BaseConstraintTests . test_custom_violation_code_message . test_custom_violation_code_message \ constraints . tests . BaseConstraintTests . test_deconstruction . test_deconstruction \ constraints . tests . CheckConstraintTests . test_eq . test_eq \ constraints . tests . CheckConstraintTests . test_repr_with_violation_error_code . test_repr_with_violation_error_code \ constraints . tests . CheckConstraintTests . test_validate_custom_error . test_validate_custom_error \ constraints . tests . UniqueConstraintTests . test_eq . test_eq \ constraints . tests . UniqueConstraintTests . test_repr_with_violation_error_code . test_repr_with_violation_error_code \ constraints . tests . UniqueConstraintTests . test_validate_conditon_custom_error . test_validate_conditon_custom_error I.3 Edit Location ⬇ { "instance_id" : "pro/golang/future-architect__vuls-54e73c2" , "detector/detector.go" : { "line_ranges" : [[134, 142], [145, 151], [154, 166]], "code_snippets" : [ "......" , "35|// Detect vulns and fill CVE detailed information" , "36|func Detect(dbclient DBClient, rs []models.ScanResult, dir string) ([]models.ScanResult, error) {" , "37|" , "38| // Use the same reportedAt for all rs" , "39| reportedAt := time.Now()" , "40| for i, r := range rs {" , "41| if !c.Conf.RefreshCve && !needToRefreshCve(r) {" , "42| logging.Log.Info(’No need to refresh’)" , "43| continue" , "44| }" , "45|" , "46| if !reuseScannedCves(&r) {" , "47| r.ScannedCves = models.VulnInfos{}" , "48| }" , "As the content is too long, parts of this example about the dependencies of the Edit Location in the same file have been omitted; the actual input provided to the agent is complete." "114| rs[i] = r" , "115| }" , "116|" , "117| // Overwrite the json file every time to clear the fields specified in config.IgnoredJSONKeys" , "118| for _, r := range rs {" , "119| if s, ok := c.Conf.Servers[r.ServerName]; ok {" , "120| r = r.ClearFields(s.IgnoredJSONKeys)" , "121| }" , "122| //TODO don’t call here" , "123| if err := reporter.OverwriteJSONFile(dir, r); err != nil {" , "124| return nil, xerrors.Errorf(’Failed to write JSON: %w’, err)" , "125| }" , "126| }" , "127|" , "128| if c.Conf.DiffPlus || c.Conf.DiffMinus {" , "129| prevs, err := loadPrevious(rs)" , "130| if err != nil {" , "131| return nil, err" , "132| }" , "133| rs = diff(rs, prevs, c.Conf.DiffPlus, c.Conf.DiffMinus)" , "134| }" , "135|" , "136| for i, r := range rs {" , "137| r = r.FilterByCvssOver(c.Conf.CvssScoreOver)" , "138| r = r.FilterUnfixed(c.Conf.IgnoreUnfixed)" , "139| r = r.FilterInactiveWordPressLibs(c.Conf.WpScan.DetectInactive)" , "140|" , "141| // IgnoreCves" , "142| ignoreCves := []string{}" , "143| if r.Container.Name == ’’ {" , "144| ignoreCves = c.Conf.Servers[r.ServerName].IgnoreCves" , "145| } else if con, ok := c.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok {" , "146| ignoreCves = con.IgnoreCves" , "147| }" , "148| r = r.FilterIgnoreCves(ignoreCves)" , "149|" , "150| // ignorePkgs" , "151| ignorePkgsRegexps := []string{}" , "152| if r.Container.Name == ’’ {" , "153| ignorePkgsRegexps = c.Conf.Servers[r.ServerName].IgnorePkgsRegexp" , "154| } else if s, ok := c.Conf.Servers[r.ServerName].Containers[r.Container.Name]; ok {" , "155| ignorePkgsRegexps = s.IgnorePkgsRegexp" , "156| }" , "157| r = r.FilterIgnorePkgs(ignorePkgsRegexps)" , "158|" , "159| // IgnoreUnscored" , "160| if c.Conf.IgnoreUnscoredCves {" , "161| r.ScannedCves = r.ScannedCves.FindScoredVulns()" , "162| }" , "163|" , "164| rs[i] = r" , "165| }" , "166| return rs, nil" , "167|}" , "......" , "169|// DetectPkgCves detects OS pkg cves" , "170|func DetectPkgCves(dbclient DBClient, r *models.ScanResult) error {" , "171| // Pkg Scan" , "172| if r.Release != ’’ {" , "173| // OVAL" , "174| if err := detectPkgsCvesWithOval(dbclient.OvalDB, r); err != nil {" , "175| return xerrors.Errorf(’Failed to detect CVE with OVAL: %w’, err)" , "176| }" , "177|" , "178| // gost" , "179| if err := detectPkgsCvesWithGost(dbclient.GostDB, r); err != nil {" , "180| return xerrors.Errorf(’Failed to detect CVE with gost: %w’, err)" , "181| }" , "182| } else if reuseScannedCves(r) {" , "183| logging.Log.Infof(’r.Release is empty. Use CVEs as it as.’)" , "184| } else if r.Family == constant.ServerTypePseudo {" , "185| logging.Log.Infof(’pseudo type. Skip OVAL and gost detection’)" , "186| } else {" , "187| return xerrors.Errorf(’Failed to fill CVEs. r.Release is empty’)" , "188| }" , "189|" , "190| for i, v := range r.ScannedCves {" , "191| for j, p := range v.AffectedPackages {" , "192| if p.NotFixedYet && p.FixState == ’’ {" , "193| p.FixState = ’Not fixed yet’" , "194| r.ScannedCves[i].AffectedPackages[j] = p" , "195| }" , "196| }" , "197| }" , "198|" , "199| // To keep backward compatibility" , "200| // Newer versions use ListenPortStats," , "201| // but older versions of Vuls are set to ListenPorts." , "202| // Set ListenPorts to ListenPortStats to allow newer Vuls to report old results." , "203| for i, pkg := range r.Packages {" , "204| for j, proc := range pkg.AffectedProcs {" , "205| for _, ipPort := range proc.ListenPorts {" , "206| ps, err := models.NewPortStat(ipPort)" , "207| if err != nil {" , "208| logging.Log.Warnf(’Failed to parse ip:port: %s, err:%+v’, ipPort, err)" , "209| continue" , "210| }" , "211| r.Packages[i].AffectedProcs[j].ListenPortStats = append(" , "212| r.Packages[i].AffectedProcs[j].ListenPortStats, *ps)" , "213| }" , "214| }" , "215| }" , "216|" , "217| return nil" , "218|}" , "......" , ] }, "detector/wordpress.go" : { "line_ranges" : [[110, 116]], "code_snippets" : [ "......" , "51|// DetectWordPressCves access to wpscan and fetch scurity alerts and then set to the given ScanResult." , "52|// https://wpscan.com/" , "53|func detectWordPressCves(r *models.ScanResult, cnf *c.WpScanConf) (int, error) {" , "54| if len(r.WordPressPackages) == 0 {" , "55| return 0, nil" , "56| }" , "57| // Core" , "58| ver := strings.Replace(r.WordPressPackages.CoreVersion(), ’.’, ’’, -1)" , "59| if ver == ’’ {" , "60| return 0, errof.New(errof.ErrFailedToAccessWpScan," , "61| fmt.Sprintf(’Failed to get WordPress core version.’))" , "62| }" , "63| url := fmt.Sprintf(’https://wpscan.com/api/v3/wordpresses/%s’, ver)" , "64| wpVinfos, err := wpscan(url, ver, cnf.Token)" , "65| if err != nil {" , "66| return 0, err" , "67| }" , "68|" , "69| // Themes" , "70| themes := r.WordPressPackages.Themes()" , "71| if !cnf.DetectInactive {" , "72| themes = removeInactives(themes)" , "73| }" , "74| for _, p := range themes {" , "75| url := fmt.Sprintf(’https://wpscan.com/api/v3/themes/%s’, p.Name)" , "76| candidates, err := wpscan(url, p.Name, cnf.Token)" , "77| if err != nil {" , "78| return 0, err" , "79| }" , "80| vulns := detect(p, candidates)" , "81| wpVinfos = append(wpVinfos, vulns...)" , "82| }" , "83|" , "84| // Plugins" , "85| plugins := r.WordPressPackages.Plugins()" , "86| if !cnf.DetectInactive {" , "87| plugins = removeInactives(plugins)" , "88| }" , "89| for _, p := range plugins {" , "90| url := fmt.Sprintf(’https://wpscan.com/api/v3/plugins/%s’, p.Name)" , "91| candidates, err := wpscan(url, p.Name, cnf.Token)" , "92| if err != nil {" , "93| return 0, err" , "94| }" , "95| vulns := detect(p, candidates)" , "96| wpVinfos = append(wpVinfos, vulns...)" , "97| }" , "98|" , "99| for _, wpVinfo := range wpVinfos {" , "100| if vinfo, ok := r.ScannedCves[wpVinfo.CveID]; ok {" , "101| vinfo.CveContents[models.WpScan] = wpVinfo.CveContents[models.WpScan]" , "102| vinfo.VulnType = wpVinfo.VulnType" , "103| vinfo.Confidences = append(vinfo.Confidences, wpVinfo.Confidences...)" , "104| vinfo.WpPackageFixStats = append(vinfo.WpPackageFixStats, wpVinfo.WpPackageFixStats...)" , "105| r.ScannedCves[wpVinfo.CveID] = vinfo" , "106| } else {" , "107| r.ScannedCves[wpVinfo.CveID] = wpVinfo" , "108| }" , "109| }" , "110| return len(wpVinfos), nil" , "111|}" , "112|" , "113|func wpscan(url, name, token string) (vinfos []models.VulnInfo, err error) {" , "114| body, err := httpRequest(url, token)" , "115| if err != nil {" , "116| return nil, err" , "117| }" , "118| if body == ’’ {" , "119| logging.Log.Debugf(’wpscan.com response body is empty. URL: %s’, url)" , "120| }" , "121| return convertToVinfos(name, body)" , "122|}" , "......" , "124|func detect(installed models.WpPackage, candidates []models.VulnInfo) (vulns []models.VulnInfo) {" , "125| for _, v := range candidates {" , "126| for _, fixstat := range v.WpPackageFixStats {" , "127| ok, err := match(installed.Version, fixstat.FixedIn)" , "128| if err != nil {" , "129| logging.Log.Warnf(’Failed to compare versions %s installed: %s, fixedIn: %s, v: %+v’," , "130| installed.Name, installed.Version, fixstat.FixedIn, v)" , "131| // continue scanning" , "132| continue" , "133| }" , "134| if ok {" , "135| vulns = append(vulns, v)" , "136| logging.Log.Debugf(’Affected: %s installed: %s, fixedIn: %s’," , "137| installed.Name, installed.Version, fixstat.FixedIn)" , "138| } else {" , "139| logging.Log.Debugf(’Not affected: %s : %s, fixedIn: %s’," , "140| installed.Name, installed.Version, fixstat.FixedIn)" , "141| }" , "142| }" , "143| }" , "144| return" , "145|}" , "......" , ] } } I.4 API Usage ⬇ { "instance_id" : "sphinx-doc__sphinx-7462" , "sphinx/domains/python.py" : [ { "file of api definition" : "/testbed/sphinx/domains/python.py" , "lineno of api definition" : [92, 111], "api function name" : "unparse" , "api function should be called in this way" : "unparse(elem)" , "the approximate lineno where this api appear in the file you need to modify" : 111, "api function definition" : [ "def _parse_annotation(annotation: str) -> List[Node]:" , " ’’’Parse type annotation.’’’" , " def make_xref(text: str) -> addnodes.pending_xref:" , " if text == ’None’:" , " reftype = ’obj’" , " else:" , " reftype = ’class’" , "" , " return pending_xref(’’, nodes.Text(text)," , " refdomain=’py’, reftype=reftype, reftarget=text)" , "" , " def unparse(node: ast.AST) -> List[Node]:" , " if isinstance(node, ast.Attribute):" , " return [nodes.Text(’%s.%s’ % (unparse(node.value)[0], node.attr))]" , " elif isinstance(node, ast.Expr):" , " return unparse(node.value)" , " elif isinstance(node, ast.Index):" , " return unparse(node.value)" , " elif isinstance(node, ast.List):" , " result = [addnodes.desc_sig_punctuation(’’, ’[’)] # type: List[Node]" , " for elem in node.elts:" , " result.extend(unparse(elem))" , " result.append(addnodes.desc_sig_punctuation(’’, ’, ’))" , " result.pop()" , " result.append(addnodes.desc_sig_punctuation(’’, ’]’))" , " return result" , " elif isinstance(node, ast.Module):" , " return sum((unparse(e) for e in node.body), [])" , " elif isinstance(node, ast.Name):" , " return [nodes.Text(node.id)]" , " elif isinstance(node, ast.Subscript):" , " result = unparse(node.value)" , " result.append(addnodes.desc_sig_punctuation(’’, ’[’))" , " result.extend(unparse(node.slice))" , " result.append(addnodes.desc_sig_punctuation(’’, ’]’))" , " return result" , " elif isinstance(node, ast.Tuple):" , " if node.elts:" , " result = []" , " for elem in node.elts:" , " result.extend(unparse(elem))" , " result.append(addnodes.desc_sig_punctuation(’’, ’, ’))" , " result.pop()" , " else:" , " result = [addnodes.desc_sig_punctuation(’’, ’(’)," , " addnodes.desc_sig_punctuation(’’, ’)’)]" , "" , " return result" , " else:" , " raise SyntaxError # unsupported syntax" , "" , " try:" , " tree = ast_parse(annotation)" , " result = unparse(tree)" , " for i, node in enumerate(result):" , " if isinstance(node, nodes.Text):" , " result[i] = make_xref(str(node))" , " return result" , " except SyntaxError:" , " return [make_xref(annotation)]" , ] }, { "file of api definition" : "/opt/miniconda3/envs/testbed/lib/python3.9/site-packages/docutils/nodes.py" , "lineno of api definition" : [692, 692], "api function name" : "append" , "api function should be called in this way" : "result.append(addnodes.desc_sig_punctuation(’’, ’, ’))" , "the approximate lineno where this api appear in the file you need to modify" : 112, "api function definition" : [ "class Element(Node):" , "......" , " def append(self, item):" , " self.setup_child(item)" , " self.children.append(item)" , ] } ], "sphinx/ext/autodoc/init.py" : [ { "file of api definition" : "/testbed/sphinx/util/inspect.py" , "lineno of api definition" : [ 393, 398 ], "api function name" : "isproperty" , "api function should be called in this way" : "inspect.isproperty(member)" , "the approximate lineno where this api appear in the file you need to modify" : 2665, "api function definition" : [ "def isproperty(obj: Any) -> bool:" , " ’’’Check if the object is property.’’’" , " if sys.version_info >= (3, 8):" , " from functools import cached_property # cached_property is available since py3.8" , " if isinstance(obj, cached_property):" , " return True" , "" , " return isinstance(obj, property)" , ] }, { "file of api definition" : "/testbed/sphinx/events.py" , "lineno of api definition" : [ 89, 89 ], "api function name" : "emit" , "api function should be called in this way" : "self.env.app.events.emit(" source - read \ ", docname, arg)" , "the approximate lineno where this api appear in the file you need to modify" : 392, "api function definition" : [ "class EventManager:" , "......" , " def emit(self, name: str, *args: Any," , " allowed_exceptions: tuple[type[Exception], ...] = ()) -> list:" , " ’’’Emit a Sphinx event.’’’" , "" , " # not every object likes to be repr()’d (think" , " # random stuff coming via autodoc)" , " with contextlib.suppress(Exception):" , " logger.debug(’[app] emitting event: %r%s’, name, repr(args)[:100])" , "" , " results = []" , " listeners = sorted(self.listeners[name], key=attrgetter(’priority’))" , " for listener in listeners:" , " try:" , " results.append(listener.handler(self.app, *args))" , " except allowed_exceptions:" , " # pass through the errors specified as allowed_exceptions" , " raise" , " except SphinxError:" , " raise" , " except Exception as exc:" , " if self.app.pdb:" , " # Just pass through the error, so that it can be debugged." , " raise" , " modname = safe_getattr(listener.handler, ’module’, None)" , " raise ExtensionError(__(’Handler %r for event %r threw an exception’) %" , " (listener.handler, name), exc, modname=modname) from exc" , " return results" , ] } ] } I.5 Execution Context instance_id: live/python-attrs__attrs-1319 | | | |