软件测试. 04 缺陷跟踪与 Bug 仓库研究

Size: px

Start display at page:

Download "软件测试. 04 缺陷跟踪与 Bug 仓库研究"

Samson Morrison
5 years ago
Views:

1 软件测试 04 缺陷跟踪与 Bug 仓库研究玄跻峰武汉大学计算机学院 URL: 1

2 本次课程内容软件缺陷跟踪 - 回顾白盒测试 - 缺陷跟踪系统 - 软件 bug 仓库相关研究简述 2

3 白盒测试回顾 3

4 回顾假期前的课程 - 第 01 次课, 软件测试基础 - 第 02 次课, 概念与黑盒测试 - 第 03 次课, 白盒测试 4

5 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; } public TextEditor ( ) { _menu =... } public int display ( ) {... } public void addoneplugin (Plugin plugin) {... } 5

6 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; } public TextEditor ( ) { _menu =... } public int intdisplay display ( )( {) {... } public void addoneplugin (Plugin plugin) {... } if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 6

$回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; 语句覆盖条件覆盖分支覆盖路径覆盖 } public TextEditor ( ) { _menu =... } public int intdisplay display ( )( {) {.$

7 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; 语句覆盖条件覆盖分支覆盖路径覆盖 } public TextEditor ( ) { _menu =... } public int intdisplay display ( )( {) {... } public void addoneplugin (Plugin plugin) {... } if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 7

8 public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 8

9 public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 9

10 public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; waitforclickingexit ();... }! (_menu == null && _toolbar == null) &&! (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) (_menu!= null _toolbar!= null ) && (_menu == null _toolbar == null! _exitbutton.isclicked() ) _menu!= null && _menu == null _menu!= null && _toolbar == null _menu!= null &&! _exitbutton.isclicked() _toolbar!= null && _menu == null _toolbar!= null && _toolbar == null _toolbar!= null &&! _exitbutton.isclicked() 10

11 public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; waitforclickingexit ();... }! (_menu == null && _toolbar == null) &&! (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) (_menu!= null _toolbar!= null ) && (_menu == null _toolbar == null! _exitbutton.isclicked() ) _menu!= null && _menu == null _menu!= null && _toolbar == null _menu!= null &&! _exitbutton.isclicked() _toolbar!= null && _menu == null _toolbar!= null && _toolbar == null _toolbar!= null &&! _exitbutton.isclicked() 11

12 public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null boolean isclicked ( ) { && _exitbutton.isclicked() ) return EXIT; waitforclickingexit ();... } if (_menu!= null && _toolbar!= null) throw new Exception(); return clicked; }! (_menu == null && _toolbar == null) &&! (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) (_menu!= null _toolbar!= null ) && (_menu == null _toolbar == null! _exitbutton.isclicked() ) _menu!= null && _menu == null _menu!= null && _toolbar == null _menu!= null &&! _exitbutton.isclicked() _toolbar!= null && _menu == null _toolbar!= null && _toolbar == null _toolbar!= null &&! _exitbutton.isclicked() 12

$public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null boolean isclicked ( ) { && _exitbutton.$

(_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) (_menu!= null _toolbar!= null ) && (_menu == null _toolbar == null! _exitbutton.isclicked() ) _menu!

13 public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null boolean isclicked ( ) { && _exitbutton.isclicked() ) return EXIT; waitforclickingexit ();... } if (_menu!= null && _toolbar!= null) throw new Exception(); return clicked; }! (_menu == null && _toolbar == null) &&! (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) (_menu!= null _toolbar!= null ) && (_menu == null _toolbar == null! _exitbutton.isclicked() ) _menu!= null && _menu == null _menu!= null && _toolbar == null _menu!= null &&! _exitbutton.isclicked() _toolbar!= null && _menu == null _toolbar!= null && _toolbar == null _toolbar!= null &&! _exitbutton.isclicked() 13

14 public int display ( ) { } if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null boolean isclicked ( ) { && _exitbutton.isclicked() ) return EXIT; waitforclickingexit ();... } if (_menu!= null && _toolbar!= null) throw new Exception(); return clicked; _menu!= null && _toolbar == null _toolbar!= null && _menu == null 14

15 public int display ( ) { } if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null boolean isclicked ( ) { && _exitbutton.isclicked() ) return EXIT; waitforclickingexit ();... } if (_menu!= null && _toolbar!= null) throw new Exception(); return clicked; _menu!= null && _toolbar == null _toolbar!= null && _menu == null 人工手写测试用例或用算法求解上面的表达式 15

16 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; _menu!= null && _toolbar == null _toolbar!= null && _menu == null public TextEditor ( ) { _menu =... } public int display ( ) {... } public void addoneplugin (Plugin plugin) {... } } public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 16

17 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; public TextEditor ( ) { _menu =... } public int display ( ) {... } public void addoneplugin (Plugin plugin) {... } } public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 17

18 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; public TextEditor ( ) { _menu =... } public int display ( ) {... } public void addoneplugin (Plugin plugin) {... } } public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; addoneplugin (new Plugin() ); if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 18

19 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; _menu!= null && _toolbar == null _toolbar!= null && _menu == null public TextEditor ( ) { _menu =... } public int display ( ) {... } public void addoneplugin (Plugin plugin) {... } } public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; addoneplugin (new Plugin() ); if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 19

20 回顾白盒测试 /* Class under test */ class TextEditor { Menu _menu; ToolBar _toolbar; ExitButton _exitbutton; X _menu!= null && _toolbar == null _toolbar!= null && _menu == null public TextEditor ( ) { _menu =... } public int display ( ) {... } public void addoneplugin (Plugin plugin) {... } } public int display ( ) { if (_menu == null && _toolbar == null) throw new NullPointerException; addoneplugin (new Plugin() ); if (_menu!= null && _toolbar!= null && _exitbutton.isclicked() ) return EXIT; } waitforclickingexit ();... 20

21 小结即使人工填写, 达到某个覆盖准则的白盒测试的测试用例也不容易完成 21

22 软件缺陷追踪 22

23 引子场景我家里的宽带故障, 打电话给电信总部报修过了一会本地的维修人员上门但接电话的人和维修人员可能不在一个城市这个流程是如何完成的? 23

24 引子场景我家里的宽带故障, 打电话给电信总部报修过了一会本地的维修人员上门但接电话的人和维修人员可能不在一个城市这个流程是如何完成的? 武昌区维修人员洪山区维修人员客户电信电话接线员江汉区维修人员 24

25 引子场景我家里的宽带故障, 打电话给电信总部报修过了一会本地的维修人员上门但接电话的人和维修人员可能不在一个城市这个流程是如何完成的? 25

26 Bugs Reporting a bug How to report to software company when the end users meet a bug? A bug repository (bug tracking system) A bug report, => Microsoft Windows XP

27 Bugzilla The most popular bug repository, Bugzilla.

28 Nutella Mozilla Bugzilla 28

29 Google Android Code Name Code name Version Oreo 8.0 Nougat 7.0 Marshmallow 6.0 Lollipop 5.1 Lollipop 5.0 KitKat Jelly Bean 4.3.x Jelly Bean 4.2.x Jelly Bean 4.1.x Ice Cream Sandwich Ice Cream Sandwich Code name Version Honeycomb 3.2.x Honeycomb 3.1 Honeycomb 3.0 Gingerbread Gingerbread Froyo 2.2.x Eclair 2.1 Eclair Eclair 2.0 Donut 1.6 Cupcake 1.5 (no code name) 1.1 (no code name)

30 Sample Bugzilla Bug Report Bug report image Overlay the triage questions Assigned To:? Duplicate? Reproducible? A. E. Hassan and Bugzilla: open source bug tracking tool [Anvik et al. 06]

31 Bugzilla 的使用 31

32 What is Bugzilla? Bugzilla is a bug- or issue-tracking system. Bug-tracking systems allow individual or groups of developers effectively to keep track of outstanding problems with their product. An Open Source product covered by the Mozilla Public License Written in Perl, uses MySQL database De-facto standard defect-tracking system against which all others are measured See: Thanks to Dr. Mikko Rusama with the Helsinki University of Technology for sharing the slides. 32

33 Who is using Bugzilla? Netscape/AOL Mozilla.org NASA Red Hat Software SuSe Corp The Horde Project AbiSource Real Time Enterprises, Inc Eggheads.org Strata Software RockLinux Creative Labs (makers of SoundBlaster) The Apache Foundation The Gnome Foundation Ximian Linux-Mandrake 33

34 Before entering a bug Make sure the bug has not been previously reported! Use the Bugzilla Query Form For more information, see the tutorial of how to find reported bugs. Next, be sure that you've reproduced your bug using the latest build released Development process may produce new builds even daily, and the bug you've found may already have been fixed. 34

35 A useful bug report has two qualities 1.Reproducible If an engineer can't see it or conclusively prove that it exists, the engineer will probably stamp it "WORKSFORME" or "INVALID", and move on to the next bug. Every relevant detail you can provide helps. 2.Specific The quicker the engineer can isolate the issue to a specific problem, the more likely it'll be expediently fixed. If a programmer or tester has to decipher a bug, they may spend more time cursing the submitter than solving the problem In testing WWW pages, try to isolate what on the page is triggering the crash, and include it as an HTML snippet in the bug report if possible. 35

36 A useful bug report Useful bug reports are ones that get bugs fixed! Be non-judgmental in reporting bugs. Bug reports need to be non-judgmental, non-personal and noninflammatory. Reports should be written against the product, not the person, and state only the facts. 36

37 Getting started Creating A Bug Report From the Bugzilla main page, choose "Enter a new bug report". T If you haven't logged into Bugzilla already, you'll need to enter your address, password, and press the "Login" button. Userid is your address (in most cases, your HUT unix account name Password is the same as in the course return system 37

40 Where did you find the bug?

41 How important is the bug?

42 Who will be following up on the bug?

43 What can you tell the engineer about the bug?

44 Who is allowed to view this bug?

45 Submit! Create a template?

46 Where did you find the bug? Product - In which product did you find the bug? Version - In which product version did you find the bug? Component - In which component does the bug exist? Platform - On which hardware platform did you find this bug? OS - On which Operating System (OS) did you find this bug? 46

47 Where did you find the bug? Product - In which product did you find the bug? In this course, group name is same as the product name Version - In which product version did you find the bug? We're not yet using this field. Just leave the default value as you found it. Component - In which component does the bug exist? By default, your product has only the default component Click on the Component link to see descriptions of each component. Platform - On which hardware platform did you find this bug? e.g. Macintosh, SGI, Sun, PC,... If you know the bug happens on all hardware platforms, choose 'All'. Otherwise, select the platform that you found the bug on, or "Other" if your platform isn't listed. OS - On which Operating System (OS) did you find this bug? e.g. Linux, Windows NT, Mac OS 8.5. If you know the bug happens on all OSs, choose 'All'. Otherwise, select the OS that you found the bug on, or "Other" if your OS isn't listed. 47

48 Severity - How important is the bug? Severity: How damaging is the bug? The impact of a bug. This item defaults to unspecified'. To determine the most appropriate severity for a particular bug, click on the Severity link for a full explanation of each choice, from Critical to Enhancement. You should specify what severities you are planning to use 48

49 Severity Values Unspecified the default value, severity not specified Blocker - Blocks development and/or testing work Critical - crashes, loss of data, severe memory leak Major - major loss of function Minor - minor loss of function, or other problem where easy workaround is present Trivial - cosmetic problem like misspelled words or misaligned text Enhancement - Request for enhancement, ideas 49

50 Priority This field describes the importance and order in which a bug should be fixed. Utilized by the managers and programmers or engineers to prioritize their work to be done. The available priorities are: P1 Most important P2 P3 P4 P5 Least important 50

51 Who will be following up on the bug? Assigned To - Which engineer should be responsible for fixing this bug? Cc - Who else should receive updates on changes to this bug? You would not normally change either of these fields from their default values! In this course default component owner is randomly chosen, so you may need to change the values. 51

52 Who will be following up on the bug? Assigned To: Which engineer should be responsible for fixing this bug? Bugzilla will automatically assign the bug to a default engineer upon submitting a bug report; the text box exists to allow you to manually assign it to a different engineer. Default owner is one of the students (the first one listed in the CSV input file) To see the list of default engineers for each component, click on the Component link. Every time this field changes, the status changes to NEW to make it easy to see which new bugs have appeared on a person's list Cc: Who else should receive updates on changes to this bug? List the full addresses of other individuals who should receive an update upon every change to the bug report. You can enter as many addresses as you'd like; addresses must be separated by commas, with no spaces between the addresses. 52

53 To change settings page footer prefs link settings 53

54 What can you tell about the bug? URL - On what URL did you discover this bug? Summary - How would you describe the bug, in approximately 60 or fewer characters? Description - What else can you tell the engineer about this bug? 54

55 What can you tell about the bug? URL - On what URL did you discover this bug? If you encountered the bug on a particular URL, please provide it (or, them) here. If you've isolated the bug to a specific HTML snippet, please also provide a URL for that, too or, preferably, return to the bug after you've submitted it and add the HTML snippet as an attachment. Summary - How would you describe the bug, in approximately 60 or fewer characters? A good summary should quickly and uniquely identify a bug report. Otherwise, developers cannot meaningfully query by bug summary, and will often fail to pay attention to your bug report when reviewing a 10 page bug list. Think of it as a "title". A summary of "Drag-scrolling any web page crashes Mac builds" is a useful title. "Crash" or "Drag Crash" would be examples of a bad title. 55

56 What can you tell about the bug? - Description What else can you tell the engineer about this bug? Provide as detailed of a problem diagnosis in this field as possible, including as much as possible of the following information: Overview Description - More detailed expansion of summary. Steps to Reproduce - The minimal set of steps necessary to trigger the bug. Include any special setup steps. Actual Results - What the application did after performing the above steps. Expected Results - What the application should have done, were the bug not present. Build Date & Platform - Date and platform of the build that you first encountered the bug in. Additional Builds and Platforms - Whether or not the bug takes place on other platforms or browsers. Additional Information - Minimized HTML snippets, Talkback crash IDs, and any other debugging information. (ATTACHEMENT) 56

57 SoberIT Software Business and Engineering Institute Bug life-cycle Status and Resolution 57

58 Bug Life-Cycle Unconfirmed Transition is allowed from any open state to the Resolved state New Assigned Resolved Verified Reassign Open States Reopened Closed End States

59 Bug Status Open States NEW - This bug has recently been added to the assignee's list of bugs and must be processed. Bugs in this state may be accepted, and become ASSIGNED, passed on to someone else, and remain NEW, or resolved and marked RESOLVED. ASSIGNED - This bug is not yet resolved, but is assigned to someone who thinks they can fix it. From here bugs can be given to another person and become NEW, or resolved and become RESOLVED. REOPENED - The bug was once resolved, but the resolution was deemed incorrect. For example, a WORKSFORME bug is REOPENED when more information shows up and the bug is now reproducible. From here bugs are either marked ASSIGNED or RESOLVED. 59

60 Bug Status Unconfirmed State UNCONFIRMED - Nobody has validated that this bug needs to be fixed. Users who have the correct permissions may confirm this bug, changing its state to NEW. A bug may be directly resolved and marked RESOLVED but usually a bug will be confirmed by the person to whom it is assigned. Usually, an UNCONFIRMED bug will be left unconfirmed until someone has verified that the bug the reporter submitted actually occurs. Bugzilla administrator may specify the number of votes a bug in this product needs to automatically get out of the UNCONFIRMED state. THIS BUG STATE IS NOT USED IN THIS COURSE! 60

61 Bug Status End States RESOLVED - A resolution has been made, and it is awaiting verification by the QA. From here bugs are either re-opened and become REOPENED, are marked VERIFIED, or are closed for good and marked CLOSED. VERIFIED- QA has looked at the bug and the resolution and agrees that the appropriate action has been taken. Bugs remain in this state until the product they were reported against actually ships, at which point they become CLOSED. CLOSED - The bug is considered dead, the resolution is correct, and the product the bug has been reported against is terminated or shipped. Any zombie bugs who choose to walk the earth again must do so by becoming REOPENED. This state is rarely ever used. NOTE: Resolution values can only be specified for bugs being in one of the end states! 61

62 Resolution The resolution field indicates what happened to this bug. Only bugs in Resolved state will be marked with one of the resolutions. All bugs which are in one of the Open states have no associated resolution. Bug Life-Cycle Unconfirmed Transition is allowed from any open state to the Resolved state New Assigned Resolved Verified Reassign Open States Reopened Closed End States 62

63 Resolution FIXED - A fix for this bug is checked into the tree and tested. INVALID - The problem described is not a bug. WONTFIX - The problem described is a bug which will never be fixed. LATER - The problem described is a bug which will not be fixed in this version of the product. REMIND - The problem described is a bug which will probably not be fixed in this version of the product, but might still be. DUPLICATE - The problem is a duplicate of an existing bug. Marking a bug duplicate requires the bug number of the duplicate and that number will be placed in the bug description. WORKSFORME - All attempts at reproducing this bug were futile, reading the code produces no clues as to why this behavior would occur. If more information appears later, please re-assign the bug, for now, file it. 63

64 Go to Bugzilla 64

65 软件数据分析 Software Data Analysis 65

66 科学研究发现问题 + 解决问题 66

67 科学研究发现问题 + 解决问题例如, 发现软件工程中数据暴露的问题 ; 发现软件开发中的现象 67

68 科学研究发现问题 + 解决问题例如, 发现软件工程中数据暴露的问题 ; 发现软件开发中的现象软件数据分析不是数据算法的应用而是发掘未知的现象和原因 68

and maintenance of software, and the study of these approaches; that is, the application of engineering to

69 Software Engineering The IEEE Computer Society's Software Engineering Body of Knowledge (SWEBOK) Software engineering the application of a systematic, disciplined, quantifiable approach to the development, operation, and maintenance of software, and the study of these approaches; that is, the application of engineering to software. For future Interdisciplinary General Computer Science Theory Software Engineering AI Networks [SWE]

configuration management Software engineering management Software development process

70 Software Process Research Fields in SE Software requirements Software architecture Software design Software development Software testing Software maintenance Software configuration management Software engineering management Software development process Software engineering tools Software quality Research Theory Methodology Empirical software engineering

71 Software Data [TX11] L. Tan & T. Xie. Text Analytics for Software Engineering: Applications of Natural Language Processing. Tech. Brief. FSE, 2011.

72 Software Data and Tasks

73 Software Artifacts

74 Software Artifacts, cont

75 A Large SE Data Repository Promise data repository Many kinds of software data, e.g., defects, efforts, requirements, models, etc. [Pro11]

76 [TX11] L. Tan & T. Xie. Text Analytics for Software Engineering: Applications of Natural Language Processing. Tech. Brief. FSE, 2011.

77 Overview of SE Data Analysis programming defect detection testing debugging maintenance software engineering tasks helped by data mining classification association/ patterns clustering data mining techniques code bases change history program states software engineering data structural entities [HX08] A.E. Hassan, T. Xie. Mining Software Engineering Data. Tutorial. ICSE, 2008 bug reports 77

78 Roadmap Aspects Data Algorithms Tasks [XTL09] T. Xie, S. Thummalapenta, D. Lo, C. Liu. Data Mining for Software Engineering. IEEE Software, 2009.

79 An Example of SE data [SZZ05] J. Sliwerski, T. Zimmermann, A. Zeller. Don't Program on Fridays, How to Locate Fix- Inducing Changes. WSR, 2005.

80 An Example, Four Most Risky Words Based on the statistics on Eclipse, the words in most bugs. => Defect correlation with letters

81 An Example, Four Most Risky Words If you remove them, risky may disappear [ZZB11] A. Zeller, T. Zimmermann, C. Bird. Failure is a Four-Letter Word, A Parody in Empirical Research. PROMISE, 2011.

82 缺陷跟踪与 Bug 仓库研究欢迎提问 128

Bug tracking. Second level Third level Fourth level Fifth level. - Software Development Project. Wednesday, March 6, 2013

Bug tracking. Second level Third level Fourth level Fifth level. - Software Development Project. Wednesday, March 6, 2013 Bug tracking Click to edit Master CSE text 2311 styles - Software Development Project Second level Third level Fourth level Fifth level Wednesday, March 6, 2013 1 Prototype submission An email with your