diff --git a/project_task_sheets/phase_05/project_phase05_tasks/.gitignore b/project_task_sheets/phase_05/project_phase05_tasks/.gitignore new file mode 100644 index 0000000..afdad5a --- /dev/null +++ b/project_task_sheets/phase_05/project_phase05_tasks/.gitignore @@ -0,0 +1,306 @@ +Solution_Phase*_*.pdf +Solution_Phase*_*.zip +*.result.txt +output.txt + +## Core latex/pdflatex auxiliary files: +*.aux +*.lof +*.log +*.lot +*.fls +*.out +*.toc +*.fmt +*.fot +*.cb +*.cb2 +.*.lb + +## Intermediate documents: +*.dvi +*.xdv +*-converted-to.* +# these rules might exclude image files for figures etc. +# *.ps +# *.eps +# *.pdf + +## Generated if empty string is given at "Please type another file name for output:" +.pdf + +## Bibliography auxiliary files (bibtex/biblatex/biber): +*.bbl +*.bcf +*.blg +*-blx.aux +*-blx.bib +*.run.xml + +## Build tool auxiliary files: +*.fdb_latexmk +*.synctex +*.synctex(busy) +*.synctex.gz +*.synctex.gz(busy) +*.pdfsync + +## Build tool directories for auxiliary files +# latexrun +latex.out/ + +## Auxiliary and intermediate files from other packages: +# algorithms +*.alg +*.loa + +# achemso +acs-*.bib + +# amsthm +*.thm + +# beamer +*.nav +*.pre +*.snm +*.vrb + +# changes +*.soc + +# comment +*.cut + +# cprotect +*.cpt + +# elsarticle (documentclass of Elsevier journals) +*.spl + +# endnotes +*.ent + +# fixme +*.lox + +# feynmf/feynmp +*.mf +*.mp +*.t[1-9] +*.t[1-9][0-9] +*.tfm + +#(r)(e)ledmac/(r)(e)ledpar +*.end +*.?end +*.[1-9] +*.[1-9][0-9] +*.[1-9][0-9][0-9] +*.[1-9]R +*.[1-9][0-9]R +*.[1-9][0-9][0-9]R +*.eledsec[1-9] +*.eledsec[1-9]R +*.eledsec[1-9][0-9] +*.eledsec[1-9][0-9]R +*.eledsec[1-9][0-9][0-9] +*.eledsec[1-9][0-9][0-9]R + +# glossaries +*.acn +*.acr +*.glg +*.glo +*.gls +*.glsdefs +*.lzo +*.lzs +*.slg +*.slo +*.sls + +# uncomment this for glossaries-extra (will ignore makeindex's style files!) +# *.ist + +# gnuplot +*.gnuplot +*.table + +# gnuplottex +*-gnuplottex-* + +# gregoriotex +*.gaux +*.glog +*.gtex + +# htlatex +*.4ct +*.4tc +*.idv +*.lg +*.trc +*.xref + +# hyperref +*.brf + +# knitr +*-concordance.tex +# TODO Uncomment the next line if you use knitr and want to ignore its generated tikz files +# *.tikz +*-tikzDictionary + +# listings +*.lol + +# luatexja-ruby +*.ltjruby + +# makeidx +*.idx +*.ilg +*.ind + +# minitoc +*.maf +*.mlf +*.mlt +*.mtc[0-9]* +*.slf[0-9]* +*.slt[0-9]* +*.stc[0-9]* + +# minted +_minted* +*.pyg + +# morewrites +*.mw + +# newpax +*.newpax + +# nomencl +*.nlg +*.nlo +*.nls + +# pax +*.pax + +# pdfpcnotes +*.pdfpc + +# sagetex +*.sagetex.sage +*.sagetex.py +*.sagetex.scmd + +# scrwfile +*.wrt + +# svg +svg-inkscape/ + +# sympy +*.sout +*.sympy +sympy-plots-for-*.tex/ + +# pdfcomment +*.upa +*.upb + +# pythontex +*.pytxcode +pythontex-files-*/ + +# tcolorbox +*.listing + +# thmtools +*.loe + +# TikZ & PGF +*.dpth +*.md5 +*.auxlock + +# titletoc +*.ptc + +# todonotes +*.tdo + +# vhistory +*.hst +*.ver + +# easy-todo +*.lod + +# xcolor +*.xcp + +# xmpincl +*.xmpi + +# xindy +*.xdy + +# xypic precompiled matrices and outlines +*.xyc +*.xyd + +# endfloat +*.ttt +*.fff + +# Latexian +TSWLatexianTemp* + +## Editors: +# WinEdt +*.bak +*.sav + +# Texpad +.texpadtmp + +# LyX +*.lyx~ + +# Kile +*.backup + +# gummi +.*.swp + +# KBibTeX +*~[0-9]* + +# TeXnicCenter +*.tps + +# auto folder when using emacs and auctex +./auto/* +*.el + +# expex forward references with \gathertags +*-tags.tex + +# standalone packages +*.sta + +# Makeindex log files +*.lpz + +# xwatermark package +*.xwm + +# REVTeX puts footnotes in the bibliography by default, unless the nofootinbib +# option is specified. Footnotes are the stored in a file with suffix Notes.bib. +# Uncomment the next line to have this generated file ignored. +#*Notes.bib diff --git a/project_task_sheets/phase_05/project_phase05_tasks/Duration.java b/project_task_sheets/phase_05/project_phase05_tasks/Duration.java new file mode 100644 index 0000000..bf2fa4e --- /dev/null +++ b/project_task_sheets/phase_05/project_phase05_tasks/Duration.java @@ -0,0 +1,59 @@ +public final class Duration { + public static final Duration ZERO = new Duration(0, 0); + static final int MINUTES_PER_HOUR = 60; + static final int SECONDS_PER_MINUTE = 60; + static final int SECONDS_PER_HOUR = SECONDS_PER_MINUTE * MINUTES_PER_HOUR; + static final long NANOS_PER_SECOND = 1000_000_000L; + + private final long seconds; + private final int nanos; + + public Duration(long seconds, int nanos) { + this.seconds = seconds; + this.nanos = nanos; + } + + @Override + public String toString() { + if (this == ZERO) { + return "PT0S"; + } + long hours = seconds / SECONDS_PER_HOUR; + int minutes = (int) ((seconds % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE); + int secs = (int) (seconds % SECONDS_PER_MINUTE); + StringBuilder buf = new StringBuilder(24); + buf.append("PT"); + if (hours != 0) { + buf.append(hours).append('H'); + } + if (minutes != 0) { + buf.append(minutes).append('M'); + } + if (secs == 0 && nanos == 0 && buf.length() > 2) { + return buf.toString(); + } + if (secs < 0 && nanos > 0) { + if (secs == -1) { + buf.append("-0"); + } else { + buf.append(secs + 1); + } + } else { + buf.append(secs); + } + if (nanos > 0) { + int pos = buf.length(); + if (secs < 0) { + buf.append(2 * NANOS_PER_SECOND - nanos); + } else { + buf.append(nanos + NANOS_PER_SECOND); + } + while (buf.charAt(buf.length() - 1) == '0') { + buf.setLength(buf.length() - 1); + } + buf.setCharAt(pos, '.'); + } + buf.append('S'); + return buf.toString(); + } +} diff --git a/project_task_sheets/phase_05/project_phase05_tasks/Makefile b/project_task_sheets/phase_05/project_phase05_tasks/Makefile new file mode 100644 index 0000000..ef3a241 --- /dev/null +++ b/project_task_sheets/phase_05/project_phase05_tasks/Makefile @@ -0,0 +1,21 @@ +phase = 05 +version = 1 +name = MichaelChen + +solutionname = Solution_Phase$(phase)_$(name) +target = $(solutionname)_V$(version).zip +package = $(solutionname).pdf Duration.java Mutation_Testing_Sample_Model.xml +latexmkflags = + +.PHONY : all dev + +all : $(target) + +dev : latexmkflags = -pvc +dev : all + +$(target) : $(package) + zip -FSr $(target) $(package) + +%.pdf : %.tex + latexmk -jobname="$*" $(latexmkflags) -pdf $< diff --git a/project_task_sheets/phase_05/project_phase05_tasks/Mutation_Testing_Sample_Model.xml b/project_task_sheets/phase_05/project_phase05_tasks/Mutation_Testing_Sample_Model.xml new file mode 100644 index 0000000..4f6a02e --- /dev/null +++ b/project_task_sheets/phase_05/project_phase05_tasks/Mutation_Testing_Sample_Model.xml @@ -0,0 +1,228 @@ + + + + // Place global declarations here. + + // As input to your method / class model, you can use either... +typedef int[-25,25] cust_t; // ... a custom integer type which is restricted to a certain value range... + +const int list_size = 4; // ...or a fixed number of given values... +int values[list_size] = {-1, 0, 1, 300}; // ... which are defined in this array ... +typedef int[0,list_size-1] index_t; // ... and are indexed inside the model via this index type. + +broadcast chan initialized; // The broadcast channel assures that the initial variable assignment in the method / class model is synchronized + // with the completion of the value generation (e.g. of InputFromRange or InputFromList) + +int x_in = 0; // These variables globally hold the initial input values to the method / class model +int y_in = 0; + + + + + // Place template instantiations here. + +SE = SetEqual(); // The instance of the original model +SE_MUT = SetEqualMutant(); // The instance of the mutated model +IN_RANGE = InputFromRange(); // Use this template in the system definition below if you want take input values from an integer range... +IN_LIST = InputFromList(); // ...or this if you want to provide a predefined list of possible / probable / common input values + +// List one or more processes to be composed into a system. + +system IN_RANGE, SE, SE_MUT; + + + A[] (SE.End && SE_MUT.End) imply (SE.x == SE_MUT.x && SE.y == SE_MUT.y) + + This formula can be used to compare the results and/or states of the original and the mutated template. +In this example, it covers the requirement that for all paths (A), all states on that individual path ([]) should satisfy that if the locations "SE.End" and "SE_MUT.End" are active (meaning that both instances reached the final state), +the x and y values of the original model template should be equal to the corresponding values of the mutated template. +(Note: "SE.End" represents the "End" location of the template instance for the method / class as named in the "System declarations" section) + + + + A[] SE.End imply (SE.x == SE.y) + + This formula covers the requirement that for all paths (A), all states on that individual path ([]) should satisfy that if the location "SE.End" is active, the x and y values of that template instance (SE.x, SE.y) must be equal. +(Note: "SE.End" represents the "End" location of the template instance for the method / class as named in the "System declarations" section) + + + + A[] SE.End imply (SE.y == y_in) + + This formula covers the requirement that for all paths (A), all states on that individual path ([]) should satisfy that if the location "SE.End" is active, the y value of the template instance (SE.y) must still be equal to the input value y_in. +Of course, you can even be more strict with this property, and adapt it so that "SE.y" should remain unchanged in every single state along the algorithm execution. +(Note: "SE.End" represents the "End" location of the template instance for the method / class as named in the "System declarations" section) + + + + diff --git a/project_task_sheets/phase_05/project_phase05_tasks/Solution_Phase05_MichaelChen.tex b/project_task_sheets/phase_05/project_phase05_tasks/Solution_Phase05_MichaelChen.tex new file mode 100644 index 0000000..fc3b04b --- /dev/null +++ b/project_task_sheets/phase_05/project_phase05_tasks/Solution_Phase05_MichaelChen.tex @@ -0,0 +1,228 @@ +\documentclass[a4paper]{scrreprt} +\usepackage[left=4cm,bottom=3cm,top=3cm,right=4cm,nohead,nofoot]{geometry} +\usepackage{graphicx} +\usepackage{tabularx} +\usepackage{listings} +\usepackage{enumitem} +\usepackage{subcaption} + +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,automata} + +\usepackage{xparse} +\usepackage{multirow} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\setlength{\textfloatsep}{16pt} + +\renewcommand{\labelenumi}{\alph{enumi})} +\renewcommand{\labelenumii}{\arabic{enumii}) } + +\newcommand{\baseinfo}[5]{ + \begin{center} + \begin{tabular}{p{15cm}r} + \vspace{-4.5pt}{ \Large \bfseries #1} & \multirow{2}{*}{} \\[0.4cm] + #2 & \\[0.5cm] + \end{tabular} + \end{center} + \vspace{-18pt}\hrule\vspace{6pt} + \begin{tabular}{ll} + \textbf{Name:} & #4\\ + \textbf{Group:} & #5\\ + \end{tabular} + \vspace{4pt}\hrule\vspace{2pt} + \footnotesize \textbf{Software Testing} \hfil - \hfil Summer 2022 \hfil - \hfil #3 \hfil - \hfil Sibylle Schupp / Sascha Lehmann \hfil \\ +} + +\newcounter{question} +\NewDocumentEnvironment{question}{m o}{% + \addtocounter{question}{1}% + \paragraph{\textcolor{red}{Task~\arabic{question}} - #1\hfill\IfNoValueTF{#2}{}{[#2 P]}} + \leavevmode\\% +}{% + \vskip 1em% +} + +\NewDocumentEnvironment{answer}{}{% + \vspace{6pt} + \leavevmode\\ + \textit{Answer:}\\[-0.25cm] + {\color{red}\rule{\textwidth}{0.4mm}} +}{% + \leavevmode\\ + {\color{red}\rule{\textwidth}{0.4mm}} +} + +\newcommand{\projectinfo}[5]{ + \baseinfo{Project Phase #1 - Submission Sheet}{#2}{#3}{#4}{#5} +} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\def\name{[Add name here]} +\def\group{[Add group here]} + +\begin{document} +\projectinfo{5}{Software Testing - Syntax Coverage\small}{\today}{\name}{\group} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Task 1 %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{question}{Answer basic questions on Syntax Coverage}[3] +\begin{enumerate}[topsep=0pt, leftmargin=*] + + \item Define the following terms in your own words: + \begin{enumerate} + \item \textit{Production Rule}, \textit{Generator}, and \textit{Terminal} + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item \textit{Mutant}, \textit{Ground String}, and \textit{Mutation Operator} + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item \textit{RIP Model} + \begin{answer} + [TODO: Add answer here] + \end{answer} + \end{enumerate} + \item Name and describe \textit{2 syntax-based coverage criteria}. Does one of these two criteria subsume the other? Explain why, or provide a counterexample. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item What does it mean to \textit{"kill a mutant"}? Explain the concept with an own code example. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Name and describe the \textit{4 different classified types} of mutants in the context of mutant killing. + \begin{answer} + [TODO: Add answer here] + \end{answer} + +\end{enumerate} +\end{question} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Task 2 %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{question}{Apply Syntax Coverage criteria to a sample program}[5] +\begin{enumerate}[topsep=0pt, leftmargin=*] + + \item Select \textit{one} mutation operator from Table~\ref{table:operators} (the operators are taken from \textit{Introduction to Software Testing} by Ammann and Offutt), and apply it \textit{once} to every occurrence of a target expression, statement or instruction inside the given \texttt{toString()} method (e.g., choose the \textit{SOR} operator, and for each \verb|<<|, \verb|>>|, or \verb|>>>| that occurs in the method, replace it with \textit{one} other shift operator). The operator has to be applicable to at least two parts of the code. For each mutation, document the line, original epression, and mutated expression. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Select \textit{one} other mutation operator, and apply all of its concrete mutations to \textit{one} target expression, statement or instruction inside the given \texttt{toString()} method (e.g., choose the \textit{SOR} operator, and replace one occurring \verb|<<| with the other shift operators \verb|>>| and \verb|>>>|). Document the line, original epression, and mutated expressions. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Pick one set of method mutations (either the mutations of task 2a or 2b), and provide a minimum set of tests that kill all introduced mutants \textit{strongly} (\textit{Strong Mutation Coverage (SMC)}). If that is not possible for a concrete mutant, explain why, and kill that mutant \textit{weakly} (\textit{Weak Mutation Coverage (WMC)}) instead. If neither of that is possible, explain it as well. + \begin{answer} + [TODO: Add answer here] + + \begin{lstlisting}[language=Java,belowskip=-0.8\baselineskip] +/* Add code here */ + \end{lstlisting} + % or: \lstinputlisting[language=Java,belowskip=-0.8\baselineskip]{file_name.java} + \end{answer} + + \item For the \textit{other} set of method mutations, provide a minimum set of tests that kill the mutants only \textit{weakly}, but \textbf{not} \textit{strongly}. If that is not possible for a concrete mutant, explain why, and kill the mutants \textit{strongly} instead. If that is not possible as well, explain it. + \begin{answer} + [TODO: Add answer here] + + \begin{lstlisting}[language=Java,belowskip=-0.8\baselineskip] +/* Add code here */ + \end{lstlisting} + % or: \lstinputlisting[language=Java,belowskip=-0.8\baselineskip]{file_name.java} + \end{answer} + + \item For both applied operators, select one concrete mutation and describe the conditions that need to hold for the \textit{Reachability}, \textit{Infection}, and \textit{Propagation} of the induced change. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \begin{table}[t] + \footnotesize + \hspace*{-1cm}\begin{tabular}{|c|c|l|} + \hline + \textbf{Abbr.} & \textbf{Mutation Operator Name} & \textbf{Mutation (\textit{a} is a constant or expression)} \\ \cline{1-3} + ABS & Absolute Value Insertion & $a \rightarrow abs(a), -abs(a), failOnZero(a)$ \\ \cline{1-3} + UOI & Unary Operator Insertion & $a \rightarrow \{!,-,+,\sim\}a$\\ \cline{1-3} + UOD & Unary Operator Deletion & $\{!,-,+,\sim\}a \rightarrow a$\\ \cline{1-3} + AOR & Arithmetic Operator Replacement & $a+b \rightarrow a \{-,*,/,**,\%\} b, a, b$\\ \cline{1-3} + ROR & Relational Operator Replacement & $a>b \rightarrow a \{>=,<,<=,==,!=\} b, true, false$\\ \cline{1-3} + COR & Conditional Operator Replacement & $a\&\&b \rightarrow a \{||,\&,|,\hat{}\} b, true, false, a, b$\\ \cline{1-3} + SOR & Shift Operator Replacement & $a<>,>>>\} b, a$\\ \cline{1-3} + LOR & Logical Operator Replacement & $a\&b \rightarrow a \{|,\hat{}\} b, a, b$\\ \cline{1-3} + ASR & Assignment Operator Replacement & $a+=b \rightarrow a \{-=,*=,/=,\%=,\&=,|=,\hat{}=,<<=,>>=,>>>=\} b$\\ \cline{1-3} + SVR & Scalar Variable Replacement & $a=b*c \rightarrow a=b*b,a=b*a,a=a*c,a=c*c,b=b*c,c=b*c$\\ \cline{1-3} + BSR & Bomb Statement Replacement & instruction $\rightarrow$ Bomb() (throws exception when reached)\\ + \hline + \end{tabular} + \caption{Exemplary mutation operators with their introduced changes for an expression / instruction} + \label{table:operators} + \end{table} + +\end{enumerate} +\end{question} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Task 3 %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{question}{Application to software project}[8] +\begin{enumerate}[topsep=0pt, leftmargin=*] + + \item As a group, decide on \textbf{one test suite} (original + own tests) that you want to use for this task + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item In consultation with your group, pick a unique mutation tool for Java code (i.e., no two members of one group should use the same tool). + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Depending on the capabilities of the selected mutation testing tool, determine the amount of \textit{killed mutants} for at least \textbf{3 different mutation operators} (handle the results individually as long as your tool returns the results for each individual operator, or supports the application of single selected operators). + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item For each of the applied mutation operators, pick a distinct method of your project that was affected by that operator, and highlight / describe the instructions that are changed. Explain how the test suite does or does not satisfy the criteria of \textit{Reachability}, \textit{Infection}, and \textit{Propagation} for that part. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Pick one suitable method (restricted to int values) or class from your project, and derive an automaton expressing its behaviour. For the creation of your automaton, use the \textit{Uppaal} model checker environment. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Formulate at least 2 formal requirements that your model / method needs to satisfy, create corresponding \textit{queries} in Uppaal, and execute them to assure that your base model satisfies them. Enable \texttt{Options $\rightarrow$ Diagnostic Trace $\rightarrow$ Some} to get a counterexample trace in case that a queried formula is not satisfied. This trace can be inspected in the \texttt{Simulator} tab. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Apply \textit{one} mutation operator from Table~\ref{table:operators} to \textit{one} single expression in your model, and re-evaluate your queries. Explain why the verification results of the model changed or remained the same. If a counterexample is created, note the test vector that kills the mutant. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Reroute one transition in your model (i.e., connect the edge to a different destination node), and re-evaluate your queries. Explain why the verification results of the model changed or remained the same. If a counterexample is created, note the test vector that kills the mutant. + \begin{answer} + [TODO: Add answer here] + \end{answer} + +\end{enumerate} +\end{question} + +\end{document} diff --git a/project_task_sheets/phase_05/project_phase05_tasks/project_phase05_tasks.pdf b/project_task_sheets/phase_05/project_phase05_tasks/project_phase05_tasks.pdf new file mode 100644 index 0000000..b2ff9f7 --- /dev/null +++ b/project_task_sheets/phase_05/project_phase05_tasks/project_phase05_tasks.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9b481b7267f52aadd57aadad3b0025cf5514a75ff331f37fd4a4441fd0a86ba +size 465038 diff --git a/project_task_sheets/phase_05/project_phase05_tasks/project_phase05_template.tex b/project_task_sheets/phase_05/project_phase05_tasks/project_phase05_template.tex new file mode 100644 index 0000000..fc3b04b --- /dev/null +++ b/project_task_sheets/phase_05/project_phase05_tasks/project_phase05_template.tex @@ -0,0 +1,228 @@ +\documentclass[a4paper]{scrreprt} +\usepackage[left=4cm,bottom=3cm,top=3cm,right=4cm,nohead,nofoot]{geometry} +\usepackage{graphicx} +\usepackage{tabularx} +\usepackage{listings} +\usepackage{enumitem} +\usepackage{subcaption} + +\usepackage{pgf} +\usepackage{tikz} +\usetikzlibrary{arrows,automata} + +\usepackage{xparse} +\usepackage{multirow} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\setlength{\textfloatsep}{16pt} + +\renewcommand{\labelenumi}{\alph{enumi})} +\renewcommand{\labelenumii}{\arabic{enumii}) } + +\newcommand{\baseinfo}[5]{ + \begin{center} + \begin{tabular}{p{15cm}r} + \vspace{-4.5pt}{ \Large \bfseries #1} & \multirow{2}{*}{} \\[0.4cm] + #2 & \\[0.5cm] + \end{tabular} + \end{center} + \vspace{-18pt}\hrule\vspace{6pt} + \begin{tabular}{ll} + \textbf{Name:} & #4\\ + \textbf{Group:} & #5\\ + \end{tabular} + \vspace{4pt}\hrule\vspace{2pt} + \footnotesize \textbf{Software Testing} \hfil - \hfil Summer 2022 \hfil - \hfil #3 \hfil - \hfil Sibylle Schupp / Sascha Lehmann \hfil \\ +} + +\newcounter{question} +\NewDocumentEnvironment{question}{m o}{% + \addtocounter{question}{1}% + \paragraph{\textcolor{red}{Task~\arabic{question}} - #1\hfill\IfNoValueTF{#2}{}{[#2 P]}} + \leavevmode\\% +}{% + \vskip 1em% +} + +\NewDocumentEnvironment{answer}{}{% + \vspace{6pt} + \leavevmode\\ + \textit{Answer:}\\[-0.25cm] + {\color{red}\rule{\textwidth}{0.4mm}} +}{% + \leavevmode\\ + {\color{red}\rule{\textwidth}{0.4mm}} +} + +\newcommand{\projectinfo}[5]{ + \baseinfo{Project Phase #1 - Submission Sheet}{#2}{#3}{#4}{#5} +} +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +\def\name{[Add name here]} +\def\group{[Add group here]} + +\begin{document} +\projectinfo{5}{Software Testing - Syntax Coverage\small}{\today}{\name}{\group} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Task 1 %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{question}{Answer basic questions on Syntax Coverage}[3] +\begin{enumerate}[topsep=0pt, leftmargin=*] + + \item Define the following terms in your own words: + \begin{enumerate} + \item \textit{Production Rule}, \textit{Generator}, and \textit{Terminal} + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item \textit{Mutant}, \textit{Ground String}, and \textit{Mutation Operator} + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item \textit{RIP Model} + \begin{answer} + [TODO: Add answer here] + \end{answer} + \end{enumerate} + \item Name and describe \textit{2 syntax-based coverage criteria}. Does one of these two criteria subsume the other? Explain why, or provide a counterexample. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item What does it mean to \textit{"kill a mutant"}? Explain the concept with an own code example. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Name and describe the \textit{4 different classified types} of mutants in the context of mutant killing. + \begin{answer} + [TODO: Add answer here] + \end{answer} + +\end{enumerate} +\end{question} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Task 2 %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{question}{Apply Syntax Coverage criteria to a sample program}[5] +\begin{enumerate}[topsep=0pt, leftmargin=*] + + \item Select \textit{one} mutation operator from Table~\ref{table:operators} (the operators are taken from \textit{Introduction to Software Testing} by Ammann and Offutt), and apply it \textit{once} to every occurrence of a target expression, statement or instruction inside the given \texttt{toString()} method (e.g., choose the \textit{SOR} operator, and for each \verb|<<|, \verb|>>|, or \verb|>>>| that occurs in the method, replace it with \textit{one} other shift operator). The operator has to be applicable to at least two parts of the code. For each mutation, document the line, original epression, and mutated expression. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Select \textit{one} other mutation operator, and apply all of its concrete mutations to \textit{one} target expression, statement or instruction inside the given \texttt{toString()} method (e.g., choose the \textit{SOR} operator, and replace one occurring \verb|<<| with the other shift operators \verb|>>| and \verb|>>>|). Document the line, original epression, and mutated expressions. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Pick one set of method mutations (either the mutations of task 2a or 2b), and provide a minimum set of tests that kill all introduced mutants \textit{strongly} (\textit{Strong Mutation Coverage (SMC)}). If that is not possible for a concrete mutant, explain why, and kill that mutant \textit{weakly} (\textit{Weak Mutation Coverage (WMC)}) instead. If neither of that is possible, explain it as well. + \begin{answer} + [TODO: Add answer here] + + \begin{lstlisting}[language=Java,belowskip=-0.8\baselineskip] +/* Add code here */ + \end{lstlisting} + % or: \lstinputlisting[language=Java,belowskip=-0.8\baselineskip]{file_name.java} + \end{answer} + + \item For the \textit{other} set of method mutations, provide a minimum set of tests that kill the mutants only \textit{weakly}, but \textbf{not} \textit{strongly}. If that is not possible for a concrete mutant, explain why, and kill the mutants \textit{strongly} instead. If that is not possible as well, explain it. + \begin{answer} + [TODO: Add answer here] + + \begin{lstlisting}[language=Java,belowskip=-0.8\baselineskip] +/* Add code here */ + \end{lstlisting} + % or: \lstinputlisting[language=Java,belowskip=-0.8\baselineskip]{file_name.java} + \end{answer} + + \item For both applied operators, select one concrete mutation and describe the conditions that need to hold for the \textit{Reachability}, \textit{Infection}, and \textit{Propagation} of the induced change. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \begin{table}[t] + \footnotesize + \hspace*{-1cm}\begin{tabular}{|c|c|l|} + \hline + \textbf{Abbr.} & \textbf{Mutation Operator Name} & \textbf{Mutation (\textit{a} is a constant or expression)} \\ \cline{1-3} + ABS & Absolute Value Insertion & $a \rightarrow abs(a), -abs(a), failOnZero(a)$ \\ \cline{1-3} + UOI & Unary Operator Insertion & $a \rightarrow \{!,-,+,\sim\}a$\\ \cline{1-3} + UOD & Unary Operator Deletion & $\{!,-,+,\sim\}a \rightarrow a$\\ \cline{1-3} + AOR & Arithmetic Operator Replacement & $a+b \rightarrow a \{-,*,/,**,\%\} b, a, b$\\ \cline{1-3} + ROR & Relational Operator Replacement & $a>b \rightarrow a \{>=,<,<=,==,!=\} b, true, false$\\ \cline{1-3} + COR & Conditional Operator Replacement & $a\&\&b \rightarrow a \{||,\&,|,\hat{}\} b, true, false, a, b$\\ \cline{1-3} + SOR & Shift Operator Replacement & $a<>,>>>\} b, a$\\ \cline{1-3} + LOR & Logical Operator Replacement & $a\&b \rightarrow a \{|,\hat{}\} b, a, b$\\ \cline{1-3} + ASR & Assignment Operator Replacement & $a+=b \rightarrow a \{-=,*=,/=,\%=,\&=,|=,\hat{}=,<<=,>>=,>>>=\} b$\\ \cline{1-3} + SVR & Scalar Variable Replacement & $a=b*c \rightarrow a=b*b,a=b*a,a=a*c,a=c*c,b=b*c,c=b*c$\\ \cline{1-3} + BSR & Bomb Statement Replacement & instruction $\rightarrow$ Bomb() (throws exception when reached)\\ + \hline + \end{tabular} + \caption{Exemplary mutation operators with their introduced changes for an expression / instruction} + \label{table:operators} + \end{table} + +\end{enumerate} +\end{question} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Task 3 %%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\begin{question}{Application to software project}[8] +\begin{enumerate}[topsep=0pt, leftmargin=*] + + \item As a group, decide on \textbf{one test suite} (original + own tests) that you want to use for this task + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item In consultation with your group, pick a unique mutation tool for Java code (i.e., no two members of one group should use the same tool). + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Depending on the capabilities of the selected mutation testing tool, determine the amount of \textit{killed mutants} for at least \textbf{3 different mutation operators} (handle the results individually as long as your tool returns the results for each individual operator, or supports the application of single selected operators). + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item For each of the applied mutation operators, pick a distinct method of your project that was affected by that operator, and highlight / describe the instructions that are changed. Explain how the test suite does or does not satisfy the criteria of \textit{Reachability}, \textit{Infection}, and \textit{Propagation} for that part. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Pick one suitable method (restricted to int values) or class from your project, and derive an automaton expressing its behaviour. For the creation of your automaton, use the \textit{Uppaal} model checker environment. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Formulate at least 2 formal requirements that your model / method needs to satisfy, create corresponding \textit{queries} in Uppaal, and execute them to assure that your base model satisfies them. Enable \texttt{Options $\rightarrow$ Diagnostic Trace $\rightarrow$ Some} to get a counterexample trace in case that a queried formula is not satisfied. This trace can be inspected in the \texttt{Simulator} tab. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Apply \textit{one} mutation operator from Table~\ref{table:operators} to \textit{one} single expression in your model, and re-evaluate your queries. Explain why the verification results of the model changed or remained the same. If a counterexample is created, note the test vector that kills the mutant. + \begin{answer} + [TODO: Add answer here] + \end{answer} + + \item Reroute one transition in your model (i.e., connect the edge to a different destination node), and re-evaluate your queries. Explain why the verification results of the model changed or remained the same. If a counterexample is created, note the test vector that kills the mutant. + \begin{answer} + [TODO: Add answer here] + \end{answer} + +\end{enumerate} +\end{question} + +\end{document}