steveej/msc-thesis
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 4 Packages Wiki Activity Actions

context: os dev concepts 95% done

Browse source 
* still missing code snippets for the procedure call example
This commit is contained in:
steveej 2017-09-22 21:00:34 +02:00
parent 83c5540a42
commit 79a1b918d6
11 changed files with 561 additions and 465 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
src/docs/gfx/TODO-Callstacklayout.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 40 KiB

61
src/docs/gfx/TODO-Callstacklayout.svg Normal file
View file

@ -0,0 +1,61 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg xmlns="http://www.w3.org/2000/svg" height="558px" width="684px" viewBox="0 0 21590 27940">
<defs>
<linearGradient id="a" gradientUnits="userSpaceOnUse" x2="0" y2="24400" y1="26700">
<stop stop-color="#fff" offset="0"/>
<stop stop-color="#adf" offset="1"/>
</linearGradient>
</defs>
<rect height="27940" width="34250" x="-6330" fill="#fff"/>
<g stroke="#000" stroke-width="61">
<g fill="#afc">
<path d="m6370 8620v-6430h12300v6430"/>
<path d="m6370 10400v-1760h12300v1760"/>
<path d="m6370 14500v-4090h12300v4090"/>
</g>
<path d="m6370 26700v-2340h12300v2340" fill="url(#a)"/>
<g fill="#adf">
<path d="m6370 19700v-1760h12300v1760"/>
<path d="m18670 19700v4720h-12300v-4720z"/>
<path d="m6370 17900v-3510h12300v3510"/>
</g>
<g fill="none">
<path d="m5780 24400-120-9.5-111-47-98-75.9-85-107-73-120-50.6-139-37.9-155-12.6-174v-3310l-9.5-168-34.8-161-53.7-133-66-120-88-101-95-82-111-47-123-15.8 123-9.5 111-50.6 95-69 89-111 66-123 53.7-133 34.8-158 9.5-174v-3300l12.6-171 37.9-161 50.6-133 73-120 85-98 98-85 111-47 120-12.6"/>
<path d="m19200 2190 120 22 111 63 98 85 85 133 73 145 50.6 183 37.9 180 9.5 209v4090l9.5 205 41 196 47 171 75.9 145 82 133 95 85 111 60 126 25.3-126 25.3-111 60-95 85-82 133-75.9 145-47 180-41 183-9.5 209v4080l-9.5 209-37.9 196-50.6 168-73 149-85 133-98 85-111 63-120 22"/>
</g>
<path d="m1810 2190h3470"/>
<path d="m1850 8640h3470"/>
</g>
<path d="m12400 26000h188v202h-188v-202m0-582h188v202h-188v-202m0-585h188v202h-188v-202"/>
<path d="m5030 2570 752-389-752-376 231 376-231 389"/>
<path d="m5070 9040 752-389-752-376 231 376-231 389"/>
<g style="font-family:Arial,sans-serif;text-align:center;text-anchor:middle;font-size:1200px">
<g font-size="1380px">
<text y="21607" x="12468">Parameters for</text>
<text y="23238" x="12501">DrawSquare</text>
<text y="15768" x="12455">Locals of</text>
<text y="17399" x="12501">DrawSquare</text>
<text y="19211" x="12498">Return Address</text>
<text y="11972" x="12468">Parameters for</text>
<text y="13606" x="12500">DrawLine</text>
<text y="4974" x="12455">Locals of</text>
<text y="6602" x="12500">DrawLine</text>
<text y="9870" x="12498">Return Address</text>
</g>
<text y="6464" x="27050">
<tspan y="6464" x="23984">stack frame</tspan>
<tspan y="7961" x="23984">for</tspan>
<tspan y="9458" x="23984">DrawLine</tspan>
<tspan y="10955" x="23984">subroutine</tspan>
</text>
<text y="17447" x="3865">
<tspan x="563">stack frame</tspan>
<tspan y="18944" x="563">for</tspan>
<tspan y="20440" x="563">DrawSquare</tspan>
<tspan y="21937" x="563">subroutine</tspan>
</text>
<text y="8919" x="-2242">Frame Pointer</text>
<text y="2468" x="-2081">Stack Pointer</text>
<text y="1860" x="12453">top of stack</text>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 2.8 KiB

BIN
src/docs/gfx/TODO-callstack-manipulation.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 902 KiB

BIN
src/docs/gfx/TODO-heap-stack-example-program.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 42 KiB

BIN
src/docs/gfx/TODO-nlevel-paging.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 2.5 MiB

6
src/docs/glossary.tex
View file

@ -392,15 +392,15 @@
\newglossaryentry{app}{
name = software-application,
description = {%
A bundle of one or multiples \gls{program} intended to solve a specific use-case.
A bundle of one or multiple programs with a common use-case that can be run on an \gls{os}.
}
}
\newglossaryentry{task}{
name = task
, description = {%
Generic term for any unit of work.
In the context of this thesis, it may be used for any of \glsentrytext{program}, \glsentrytext{process}, \glsentrytext{thread}, \glsentrytext{app}.
Generic term for any unit of work to be executed on the.
In the context of this thesis, it may be used for any of \glsentrytext{program}, \glsentrytext{process}, \glsentrytext{thread}.
}
}

567
src/docs/parts/context/context.tex
View file

@ -99,27 +99,26 @@ The resulting executable program might merely serve its purpose, and can contain
This is especially likely using low-abstraction languages like \gls{C} and \gls{C++} for \gls{os} development, where technical mistakes and intended behavior are very difficult to distinguish.
The goal of this thesis is to find out if the \gls{Rust} \gls{compiler} is able to mitigate this specific problem.
\chapter{OS Development Concepts}
\chapter{OS Development: Concepts, Conventions, Risks}
\label{context::os-dev-concepts}
In order to protect the memory of each executed program according to \cref{context::introduction::memory-safety::def}, the \gls{os} must be designed developed, and tested carefully.
This chapter explains concepts used in \gls{os} development and points out memory-safety critical operations without getting into great detail.
This chapter explains concepts used in \gls{os} development and points out memory-safety critical operations in necessary detail.
This is done in preparation for the next \cref{context::common-mem-safety-mistakes}, which explains weaknesses that result from memory management mistakes that were made in the attempt to implement the following concepts.
Since the \gls{os} manages the system's hardware directly, many of the implementation and design choices depend on hardware design and architecture.
To bound the extent of this document, the explanations are limited to one contemporary architecture, \gls{amd64}, and further narrowed down by focusing on the operation in 64-Bit Long Mode\cite[p.~18]{AMD64Vol2}.
For a full understanding of the \gls{os}'s design, some hardware implications are included although held to a minimum in this chapter.
More hardware specific details are required are explained and explained throughout \cnameref{rnd} along with concrete \gls{os} development examples.
\section{Virtualization}
\label{context::os-dev-concepts::virtualization}
The \gls{os} is considered the lowest software layer on the system and must know the very details of the system's hardware resources and perform raw access to it.
The goal to make \gls{os} is the only software on a system that is required to have this particular knowledge, so that other \glspl{app} could run on virtually any system.
The goal is to make \gls{os} the only software on the system that is required to have this particular knowledge, so that other \glspl{app} could run on virtually any system.
\subsection{Abstraction Layers}
The \gls{os} uses virtualization\footnote{The term \textit{virtualization} within the \gls{os} the jargon can be understood as abstraction} techniques to create layers on top of architecture specific code and abstracts it in form of an \gls{api}.
The first step of \gls{virt}\footnote{The term \textit{virtualization} within the \gls{os} the jargon can be understood as abstraction} in the \gls{os} is to create a on top of architecture specific code and abstract it in form of an internal \gls{api}.
This layer abstracts at least the \gls{cpu} and memory\cite[p.~5-7]{Arpaci-Dusseau2015}.
Higher-level complex management algorithms can be implemented hardware-independently on top of this \gls{api}, making it reusable across different architectures.
The \gls{os} then provides an \gls{api} through which \glspl{app} can request access to these virtualized resources.
This allows \gls{app} developers to develop and run different programs easily and presumably safely on the \gls{os}, agnostic of the architecture.
Higher-level \glspl{proglang} allow the \gls{app} developers to develop and run different programs comfortably, concurrently and presumably safely on any hardware which is supported by the \gls{os}.
\Cref{fig:system-abstraction-layers} shows a top-down model of the abstractions layers in the system.
\usetikzlibrary{shapes.geometric, arrows}
@ -132,7 +131,7 @@ This allows \gls{app} developers to develop and run different programs easily an
1/Software Applications,
2/System Libraries,
3/OS API,
4/OS Drivers,
4/OS {Core \& Drivers},
5/{CPU, Memory, I/O, ...}
}
{
@ -154,22 +153,38 @@ This allows \gls{app} developers to develop and run different programs easily an
\draw[dashed,black,thick]
($(block 4)!0.5!(block 5)$) ++(-20ex,0) edge["Software", "Hardware"'] ++(40ex,0);
\end{tikzpicture}
\caption{System Abstraction Layers. TODO: fill user/kernel space with colors}
\caption{System Abstraction Layers}
% TODO: fill user/kernel space with colors
\label{fig:system-abstraction-layers}
\end{figure}
\FloatBarrier
\subsection{Resource Specifics}
Virtualization has different technical implications for different resource types, depending on their nature and available count.
Programs that are executed need at least the \gls{cpu} and a certain amount of memory.
Memory is a resource that a program must explicitly ask the \gls{os} for, providing a specific amount for each request.
The \gls{cpu} is generally not explicitly requested, because any instruction by the program implicitly requires the \gls{cpu} for being executed on the system at all.
\subsection{Programming Languages}
High-level \glspl{proglang} that support or include the \gls{os} \gls{api} allow the programmer to develop \glspl{app} without knowing \gls{os} or hardware internals.
This requires conventions to be defined which must then be adopted by all involved software components including and above the "OS API" layer in \cref{fig:system-abstraction-layers}.
For \gls{amd64}, the most popular calling convention is defined in the \citetitle{Matz2009}\cite{Matz2009}, which is supported by the major \gls{C} \glspl{compiler} GCC and CLANG, as well as by \gls{Rust}'s compiler.
\FloatBarrier
\subsection{Task Model}
\subsection{Memory Virtualization: Paging}
Memory is a resource that is available in large quantities, made up from units reachable by single memory address.
The size of the smallest unit depends on the hardware and is ambiguously referred to as a \textit{memory word}, which is a fixed amount of Bytes depending on the architecture.
The whole memory can be divided into arbitrary groups of addressable units.
The \gls{os} supervises this structure, and programs must request memory from the \gls{os} to gain access to memory.
A method that combines virtualization and structuring is called paging and is explained in \cref{context::os-dev-concepts::hw-supported-mm}.
The goal of memory virtualization is to use virtual addresses in \glspl{program}, which can be dynamically mapped to physical memory addresses at system runtime.
\subsection{CPU Virtualization}
The \gls{cpu} is generally not explicitly requested by \glspl{program}, because any of the program's instructions implicitly requires the \gls{cpu} for being executed on the system at all.
Thus, the \gls{os} must chose at which time it executes which \gls{program} on the \gls{cpu}.
The goal of \gls{cpu} virtualization is to seamlessly share a single \gls{cpu} among all \glspl{program} and the \gls{os} itself, without changing any of the \glspl{program}.
How this can be achieved is explained in \cnameref{context::os-dev-concepts::preemptive-multitasking}.
\subsection{Program Execution}
\label{context::os-dev-concepts::virtualization::task}
To better understand the \gls{os}'s problem of managing executed programs and their resources there needs to be a model that can be applied to this situation.
% TODO: remove or complete this graph
%\begin{tikzpicture}[->,node distance=5ex,on grid,
% every state/.style={fill=red,draw=none,circular drop shadow,text=white}]
@ -190,8 +205,31 @@ A process can begin to exist before its execution, when the \gls{os} has interna
Processes that use the same program are not to be treated differently by the \gls{os} than any other process in terms of memory-safety, and must be prevented from mutual memory access.
These processes can differ in arguments that are passed to the program their, so that their runtime behavior can differ significantly.
\subsection{Terminology}
Terms that are often used to describe various forms of executed code are often used ambiguously in various documents, manuals, and websites.
\Cref{tab:os-dev-concepts:task-terms} defines such terms and their relationships for the scope of this document.
\begin{table}
\begin{tabularx}{\textwidth}{@{}lX@{}}
\toprule
\Gls{task} & \glsentrydesc{task} \\
\Gls{program} & \glsentrydesc{program} \\
\Gls{process} & \glsentrydesc{process} \\
\Gls{thread} & \glsentrydesc{thread} \\
\Gls{procedure} & \glsentrydesc{procedure} \\
\Gls{function} & \glsentrydesc{function} \\
\Gls{job} & \glsentrydesc{job} \\
\Gls{app} & \glsentrydesc{app} \\
\bottomrule
\end{tabularx}
\caption{Definition of commonly used terms for executable code}
\label{tab:os-dev-concepts:task-terms}
\end{table}
\FloatBarrier
\subsubsection{Demo: Process $\neq$ Program}
A great example for demonstrating this difference is \textit{"\gls{bbox}. \glsentrydesc{bbox}"}.
A great example for demonstrating this difference is the program \textit{"\gls{bbox}. \glsentrydesc{bbox}"}.
Line 1 in \cref{shell::context::os-dev-concepts::program-process} shows a command that instructs the \gls{os} - \gls{LX} in this example - to execute the program \gls{bbox} three times, with different arguments each time.
The purpose is to demonstrate that the same program \gls{bbox} is instantiated thrice with completely different functionality each time, even existing simultaneously in the \gls{os}'s process list.
@ -212,33 +250,11 @@ This command consists of subsequent calls to \gls{bbox} invoking its builtin "ps
The process list is received by passing "ps" as the argument to \gls{bbox} in the third execution.
Line 2 through 4 show the three \glspl{process} of the \gls{bbox} \gls{program} with different proccess ids and their respective arguments.
\subsection{Terminology}
These terms are used ambiguously in various documents, manuals, and websites, and shall be assigned a role for the model in this document.
\Cref{tab:os-dev-concepts:task-terms} defines these terms and their relationships for the scope of this document.
\begin{table}
\begin{tabularx}{\textwidth}{@{}lX@{}}
\toprule
\Gls{task} & \glsentrydesc{task} \\
\Gls{program} & \glsentrydesc{program} \\
\Gls{process} & \glsentrydesc{process} \\
\Gls{thread} & \glsentrydesc{thread} \\
\Gls{procedure} & \glsentrydesc{procedure} \\
\Gls{function} & \glsentrydesc{function} \\
\Gls{job} & \glsentrydesc{job} \\
\bottomrule
\end{tabularx}
\caption{Definition of commonly used terms for executable code}
\label{tab:os-dev-concepts:task-terms}
\end{table}
\FloatBarrier
\section{Preemptive Multitasking}
To extend on the terminology given in \cnameref{context::os-dev-concepts::virtualization::task} multitasking are known in the \gls{os} jargon.
In this document, it is the \gls{os}'s capability of switching tasks without terminating them, effectively keeping its runtime state in a place that persists during the period of time in which the task is not actively executed on the system's \gls{cpu}.
The preemptive attribute adds the notion of switching tasks without relying on their cooperation, but instead be able to do this at any time the \gls{os} intends to do so.
\section{Preemptive Multitasking Concepts}
\label{context::os-dev-concepts::preemptive-multitasking}
Multitasking is another vague term used in the \gls{os} jargon.
In this document, it is the \gls{os}'s capability of switching processes without terminating them, effectively keeping their runtime state in a place that persists while the task is not actively executed on the system's \gls{cpu}.
The \textit{preemptive} adds to multitasking the ability of switching tasks without relying on their cooperation, but instead be able to do this at any time the \gls{os} intends to do so.
Preemptive multitasking enables a form of \gls{cpu} virtualization, as a task is not aware of being preempted and resumed.
As explained above, the \gls{cpu} resource doesn't have to be explicitly requested, as the request to execute a program implies a dependency on the \gls{cpu}..
@ -251,37 +267,38 @@ Preemptive multitasking needs not be considered during development of single-thr
Switching tasks has different technical implications for different resources types, depending on their nature and quantity.
A single \gls{cpu} system can not execute more than one program at the same time, as it runs instructions through the \gls{cpu} one-by-one, implicitly holding the program state in form of the \gls{cpu} registers, which are preserved in between the instructions, and preserved between preemptive task switches.
While it doesn't make sense for any instruction to request the \gls{cpu} per-se, there are valid use-cases for programs to request a specific amount of \gls{cpu} bandwidth within a specific amount of time to guarantee a certain amount of computing speed.
Other use-cases emerge when concurrent programs access the exact same resources and are intermixed by the \gls{os}, creating non-sequential resource usage pattern which may put the resource in an inconsistent state and lead to unexpected results for the application.
Within this document these technicalities are considered part of the application semantics and shouldn't effect the \gls{os} developer.
Within this document these technicalities are considered part of the application semantics and shouldn't affect the \gls{os} development.
Therefore it is sufficient to recognize the \gls{os}'s responsibility of cleanly switching the program in execution periodically.
Hence for this document it is sufficient to recognize the \gls{os}'s responsibility of cleanly switching the program in execution periodically.
In contrast to the \gls{cpu}, the main memory resource is available in limited but huge quantities.
Replacing the content of the memory is not necessary on a preemptive task switch, as long as the memory is not exhausted.
This has the effect that that tasks that are currently not in execution on the \gls{cpu} still own a region of main memory.
This has the effect that tasks which are currently not in execution on the \gls{cpu} still own a region of main memory.
The \gls{os} must ensure that switching tasks is done properly for all resources to prevent interference and unintended behavior.
To ensure memory safety in this scenario, all data in the memory must be protected from unintended access, according to the definition of memory safety in \cref{context::introduction::memory-safety::def}.
\subsection{Context Switching}
The context switch is the core functionality of the multitasking as it effectively switches to a different task, possibly by preempting the one that is currently running.
When the \gls{os} preempts a task, it needs to store and preserve the current task's context.
The context consists of all volatile resources that can possibly be overwritten by another task.
The context switch is the core functionality of the multitasking as it effectively switches to a different process, possibly by preempting the one that is currently running.
When the \gls{os} preempts a process, it needs to store and preserve the current process's context.
The context consists of all volatile resources that can possibly be overwritten by another process.
This is at minimum a set of \gls{cpu} registers depending on the specific architecture.
The \gls{os} stores the preempted context in a well-known and protected memory location, so that it can be restored when this task is resumed.
The \gls{os} stores the preempted context in a well-known and protected memory location, so that it can be restored when this process is resumed.
In preemptive multitasking, context switches are not considered voluntary, but rather by force.
This works by using the \gls{cpu}'s interrupt mechanism which has the ability to jump to an \gls{os} function in the event of an interrupt.
Interrupts for this use-case are usually triggered by programmed timer interrupts, occurring continuously and regularly.
The interrupt is signaled to the \gls{os} by \gls{cpu}, so the lowest level of the process switching mechanism in the \gls{os} takes place in a hardware specific interrupt handler.
The interrupt mechanism itself is part of the \gls{cpu} which is why the lowest level of the task switching mechanism in the \gls{os} is hardware dependent.
Safety could be increased if the \gls{compiler} or in a more general sense the \gls{proglang} could assist in architecture specific code.
More details on this mechanism is given in \cnameref{rnd::sysprog-conventions::ir-driven-preemptive-cs-amd64}.
More details on this mechanism is given in \cref{context::os-dev-concepts::preemptive-multitasking-amd64}, but first it is necessary to understand the involved memory management mechanisms which are explained in \cref{context::os-dev-concepts::hw-supported-mm,context::os-dev-concepts::stackheap,context::os-dev-concepts::sf-handling-amd64,context::os-dev-concepts::stackheap-combined}.
\section{Hardware-supported Memory-Paging}
\label{context::introduction::hw-supported-mm}
\label{context::os-dev-concepts::hw-supported-mm}
To improve the efficiency and safety of memory management, developers of hardware and software have been collaborating to offload virtual memory address lookup and caching from the \gls{os} software to the hardware, the \gls{cpu}'s \gls{MMU} to be specific.
A hardware-implementation of the lookup algorithm is fast, and allows rudimentary memory permission runtime-checks to protect pages by leveraging \gls{cpu}'s security rings\cite[p.~117,~p.~145]{AMD64Vol2}.
@ -304,18 +321,20 @@ This is critical for memory-safety, as the cached virtual to physical address lo
If any lookup yields a cached result which originates from a different virtual address space, the physical address is likely to belong to a memory region to which the current task shouldn't have access to.
What makes it more difficult to manage is that there are exceptions to this, e.g. when memory is intentionally shared between two processes or threads, which must be set up by the \gls{os} according to the processes requests.
\subsubsection{Page: Chunks of Single Addresses}
To avoid the need for storing a translation mapping for every single physical address, mappings are grouped into equisized regions, called \textit{page}s.
This works by encoding the offset within the page in the virtual address, together with page's index in page table.
The offset size depends on the chosen page-size, and can be calculated with the following formula, given page-size in bytes as $p$:
\subsubsection{Pages: Chunks of Smallest Addressable Unit}
To avoid the need for storing a translation mapping for every single virtual address, mappings are grouped into equisized regions, called \textit{page}s.
This works by encoding a page-offset in the virtual address, together with page's index in the page table.
The offset size depends on the chosen page-size, and can be calculated with the following formula, given page-size $p$, a power of two given in Byte:
\begin{equation}
vaddr\_offset(p) = log_2(p); p \in N %TODO: restrain that p must be a power of 2
vaddr\_offset(p) = log_2(p), p \in \mathbb{N}^2 %TODO: restrain that p must be a power of 2
\end{equation}
For example, the \gls{amd64} default page-size of 4 KiB has a 12-bit offset, which theoretically leaves the other $64-12 = 52$ bits of the virtual address for page-table indexing.
Practically there's an architectural limit of 48 bits for the virtual address.
The resulting paging hierarchy is presented in \cnameref{rnd::sysprog-conventions::paging-amd64}.
On \gls{amd64} there's an architectural limit of 48 bits for the virtual address, thus the address constellation is different than explained here.
Details on this concrete implementation follows in \cnameref{context::os-dev-concepts::hw-supported-mm::multilevel-paging-amd64}.
\subsection{Page-Faults}
\label{context::os-dev-concepts::hw-supported-mm::page-fault}
To improve the efficiency and safety of memory management, developers of hardware and software have been collaborating to offload virtual memory address lookup and caching from the \gls{os} software to the hardware, the \gls{cpu}'s \gls{MMU} to be specific.
The page-fault is a hardware-triggered, memory-safety critical event that must be handled by the \gls{os}.
It is triggered by the \gls{cpu}'s \gls{MMU} during the virtual address lookup algorithm, when an instruction uses a virtual address for which the target page is not available.
This happens for example if the indexed page is not present in main memory or has not been allocated at all.
@ -326,15 +345,17 @@ It also happens when an instruction violates a page protection, of which four ex
\item Prevent the \gls{cpu} from executing (3) non-executable pages and (4) user pages
\end{itemize}
The \gls{os} must implement the page-fault handler to deal with it accordingly e.g. finding free physical memory and map it to the page my modifying the virtual addresse's page-table entry or indicate denied access.
The \gls{os} must implement the page-fault handler to deal with it accordingly.
For example, the case of a non-existing mapping requires to find and allocate free physical memory and map it to the page by modifying the virtual addresse's page-table entry.
Or in case of protection violation it would simply indicate denied access.
\subsection{Swapping}
The finite primary memory can only hold a finite number of virtual pages, and the \gls{os} is responsible for having the required pages present.
Besides the pages that contain the page-table itself, the pages that aren't required by the current instruction might be moved to secondary memory.
Swapping pages in and out of primary memory is risky as it requires to transfer large amounts of raw memory content, but these safety analyzes exceed the scope of this thesis.
\subsection{Multi-Level Paging}
\label{context::introduction::hw-supported-mm::multilevel-paging}
\subsection{Multi-Level Paging Concept}
\label{context::os-dev-concepts::hw-supported-mm::multilevel-paging-concept}
If only one page-table per virtual address space was used that consists of $2^{52}$ page-table entries, which must at minimum store the physical address, it would require $\frac{52 * 2^{52} [Bit]}{8*1024^4 [Bit/Byte]} = 26624$ TiB of memory for each virtual address space.
Even if only a handful of additional pages were allocated and mapped, the \gls{os} would still have to allocate this huge page-table.
This vast consumption of main memory is impractical and impossible for average systems, which rarely surpass 100 GiB of main memory.
@ -428,38 +449,106 @@ Using a hierarchical translation structure allows to save significant amounts of
\end{figure}
\FloatBarrier
The details of a 4-level-hierachy paging implemented on \gls{amd64} can be found in \cnamepref{rnd::sysprog-conventions::paging-amd64}.
\subsection{Multi-Level Paging on AMD64}
\label{context::os-dev-concepts::hw-supported-mm::multilevel-paging-amd64}
On \gls{amd64} "a four-level page-translation data structure is provided to allow long-mode operating systems to translate a 64-Bit virtual-address space into a 52-Bit physical-address space."\cite[p.~18]{AMD64Vol2}.
This allows the system to only hold the \textit{PML4} table, the which is currently referenced by the \textit{Page Map Base Register (CR3)}, available in main memory.
\section{Stack And Heap}
\label{context::introduction::os-dev-concepts::stackheap}
In \gls{proglang} and \gls{os} design and literature, the terms \gls{stack} and \gls{heap} are ubiquitous.
\Cref{fig:virtual-addr-transl} shows the 64-Bit virtual address composition on \gls{amd64}, which uses four-levels of page tables.
Counter intuitively the page-tables are not called level-\textit{n}-page-table, but the levels received distinct names in \citetitle{AMD64Vol2}.
The most-significant Bits labelled as \textit{Sign Extend} are not used for addressing purposes, but must adhere the canonical address form and simply repeat the value of the most-significant implemented Bit \cite[p.~130]{AMD64Vol2}.
The least significant Bits represent the offset within the physical page.
The four groups in between are used to index the page-table at their respective level.
\begin{figure}
\centering
\includegraphics[width=\textwidth]{gfx/Virtual-to-Physical-Address-Translation-Long-Mode.png}
\caption{Virtual to Physical Address in Long Mode\cite{AMD64Vol2}}
\label{fig:virtual-addr-transl}
\end{figure}
\subsubsection{Translation Scheme 4 KiB and 2 MiB Pages}
The \gls{amd64} architecture allows configuring the page-size, two of which will be introduced in this section.
\cref{tab:page-transl-vaddr-composition} displays the virtual address composition for the 4KiB and 2MiB page-size modes on \gls{amd64}.
The direction from top to bottom in the table corresponds to most significant to least significant - left to right - in the virtual address.
The \textit{sign extension} Bits cannot be used for actual information but act as a reservation for future architectural changes.
\begin{table}
\begin{tabular}{l | c | c}
Description & Bits in 4 KiB Pages & Bits in 2 MiB Pages \\
\hline
Sign Extend & 12 & 12 \\
Page-Map-Level-4 Offeset & 9 & 9 \\
Page-Directory-Pointer Offeset & 9 & 9 \\
Page-Directory Offeset & 9 & 9 \\
Page-Table Offeset & 9 & - \\
Physical Page Offset & 9 & 21 \\
\end{tabular}
\caption{Paging on \gls{amd64}: Virtual Address Composition 4KiB/2MiB pagesizes}
\label{tab:page-transl-vaddr-composition}
\end{table}
\begin{figure}
\centering
\includegraphics[width=\textwidth]{gfx/amd64-4kb-page-translation-long-mode}
\caption{4-Kbyte Page Translation—Long Mode\cite{AMD64Vol2}}
\label{fig:4kb-page-transl}
\end{figure}
\cref{fig:4kb-page-transl} shows the detailed virtual address composition for 4 KiB pages, using four levels of page-tables.
It uses four sets of 9-Bit indices in the virtual address, one per hierarchy level, followed by the 9 Bit page-internal offset.
An alternative approach is displayed in \cref{fig:2mb-page-transl}, using 2 MiB sized pages.
It uses three sets of 9-Bit indices for the page-tables, and a 21-Bit page-internal offset.
Increasing the page-size improves speed and memory-usage and decreases the granularity.
In this specific example the hierarchy is reduced by one level of page-tables.
This reduces the amount of storage required for the page-tables in overall and causes the lookup algorithm to finish faster.
\begin{figure}
\centering
\includegraphics[width=\textwidth]{gfx/amd64-2mb-page-translation-long-mode}
\caption{2-Mbyte Page Translation—Long Mode\cite{AMD64Vol2}}
\label{fig:2mb-page-transl}
\end{figure}
The other supported page sizes, 4 MiB and 1 GiB, as well as intermixing page sizes through the different levels don't add new insight into the mechanism and don't need to be detailed here.
\section{Stack And Heap: Basic Concepts}
\label{context::os-dev-concepts::stackheap}
In \gls{proglang} and \gls{os} literature, the terms \gls{stack} and \gls{heap} are ubiquitous.
A research for their original definition wasn't conclusive, indicating that they are to be taken as concepts rather than absolutely defined methods.
They might be implemented and used differently on various architectures, \glspl{proglang} and \glspl{os}.
This sections explains the basic concepts with the tendency towards the \gls{amd64} architecture, the \gls{C} and \gls{Rust} \glspl{proglang} and their usage for bare-metal \gls{os} and \gls{LX} \gls{app} development.
The hardware manuals \citetitle{AMD64Vol1} and \citetitle{AMD64Vol2} have no mention of the word \textit{heap}, but use \textit{stack} hundreds of times, indicating that only the \gls{stack} concept is implemented in hardware.
It is comparable to computers that had physical rewindable memory tapes, which is conceptually similar to the \gls{stack} implementations in today's memory management.
Likely the first mention of the term \gls{heap} with regard to memory management is found in \citetitle{Mailloux1969}.
It was introduced featuring "dynamic storage allocation", further described to have "the ability to create and manipulate objects whose lifetimes are not so restricted. This ability implies the use of an additional area of storage, the “heap”, in which garbage-collection techniques must be used."\cite[p.~8]{Mailloux1969}
Since then, the \gls{heap} has been implemented in all major \glspl{proglang}.
This sections explains both concepts with the tendency towards the \gls{amd64} architecture, the \gls{C} and \gls{Rust} \glspl{proglang} and their usage for bare-metal \gls{os} and \gls{LX} \gls{app} development.
\subsection{Stack: Hardware-Backed Abstract Type}
\label{context::introduction::os-dev-concepts::stackheap::stack}
In summary, the \gls{stack} is a memory model for structuring contiguous memory.
\label{context::os-dev-concepts::stackheap::stack}
In summary, the \gls{stack} is a memory model for structuring a contiguous region of memory.
It grows by adding new data entries on top of each other.
According to the \gls{stack} analogy, only the topmost element can be accessed and removed, thus it behaves like a Last-In-First-Out data structure.
The hardware manuals \citetitle{AMD64Vol1} and \citetitle{AMD64Vol2} have no mention of the word \textit{heap}, but use \textit{stack} hundreds of times, indicating that \gls{stack} is implemented in hardware to some extent.
The \gls{amd64} manuals conjunctionally describe how the \gls{stack} is used and influenced by various instructions on this architecture.
Here it grows from numerically higher to numerically lower addresses, whereas the numerically highest address is called the stack bottom, and the current numerically lowest address is the stack top.
In 64-Bit long mode \gls{amd64} doesn't consider the stack to be sized.
In 64-Bit long mode on \gls{amd64}, the \gls{cpu} doesn't consider the stack to be sized or explicitly bounded, which is highly safety critical.
The \gls{stack} is typically allocated per process or thread and stores the \glspl{sf} for each procedure.
The \gls{sf} is automatically cleaned up or simply forgotten once the procedure has completed.
On chains of procedure calls, each preceding \gls{sf} remains on the stack.
\cref{TODO-Callstacklayout}\footnote{By R. S. Shaw - Own work, Public Domain, \url{https://commons.wikimedia.org/w/index.php?curid=1956587}} displays a \gls{stack} that contains two \glspl{sf} from different procedures.
The \gls{stack} is typically allocated per process or thread and is used to store each procedure's \gls{sf}.
Each \gls{sf} is automatically cleaned up or simply forgotten once the procedure has completed.
When multiple procedure calls are nested, each preceding \gls{sf} remains on the stack in front of the next one.
\cref{fig:stack-with-two-frames}\footnote{By R. S. Shaw - Own work, Public Domain, \url{https://commons.wikimedia.org/w/index.php?curid=1956587}} displays a \gls{stack} that contains two \glspl{sf} from different procedures.
In this picture, the \gls{stack} grows upwards.
\begin{figure}[ht]
\centering
%\begin{wrapfigure}{R}{0.5\textwidth}
\includegraphics[width=0.49\textwidth]{gfx/TODO-Callstacklayout}
\caption{TODO-Redraw-Callstacklayout}
\label{TODO-Callstacklayout}
\caption{An upwards growing stack with two stack-frames}
\label{fig:stack-with-two-frames}
%\end{wrapfigure}
\end{figure}
% TODO: draw stack with multiple stack-frames
@ -473,8 +562,9 @@ Additionally, the \glspl{stack} must be prevented from growing into other memory
Since \Gls{stack} management is memory-safety critical for \gls{os} developers when implementing memory management for multitasking within the \gls{os}, it is one of the main subjects in \cnameref{rnd}, \cref{rnd::existing-os-dev-with-rust,rnd::imezzos-preemptive-multitasking}.
\subsection{Heap: Organized Chaos}
\label{context::introduction::hw-supported-mm::stackheap::heap}
\Gls{heap} is an ambiguous term that names a data structure in more theoretical computer science and a memory model in system programming.
\label{context::os-dev-concepts::stackheap::heap}
\Gls{heap} is an ambiguous term that names two computer related models.
A data structure in theoretical computer science, and a memory model in system resource management.
This document refers to the latter.
The \gls{heap} is managed by the \gls{os} to keep track of allocated memory on behalf of all \glspl{app} on the system.
@ -489,7 +579,197 @@ Memory which is not cleaned up by properly is blocked until the \gls{app} is ter
If \gls{heap} allocations within the \gls{os} are not cleaned up properly, the allocated memory is lost until the whole system is restarted.
Between properly cleaning up and loosing memory allocations is a whole range of possible memory-safety issues, which are explained in \cref{context::introduction::memory-safety-violation-in-sw}.
\section{Stack Frame Handling on AMD64}
\label{context::os-dev-concepts::sf-handling-amd64}
The usage of the \gls{stack} is tightly coupled with control flow instructions in conjunction with two registers, the Stack-Frame Base Pointer (RBP) and the Stack Pointer (RSP).
The instructions that use these registers and explicitly or implicitly work with the stack\cite[p.~83]{AMD64Vol1} can be grouped into the following categories.
Together they can be used to perform \gls{stack} based procedure calls, as demonstrated in the following \cref{context::os-dev-concepts::sf-handling-amd64::procedure-call-example}.
\subsection{Direct Stack Data Management Instructions}
\mintinline{nasm}{PUSH} a takes value operand which is to be pushed onto the stack.
The address in RSP moves towards numerically lower addresses with every \mintinline{nasm}{PUSH} instruction, which stores a new data entry on top.
The order is to first change the RSP and then copy the value at its new address.
\mintinline{nasm}{POP} takes a storage reference operand - \gls{cpu} register or memory address.
It works in the opposite direction to \mintinline{nasm}{PUSH}.
First, consuming the top-most data entry and storing it on the operand location, then moving the RSP address towards the numerically higher RBP address.
When RBP and RSP point to the same address, the stack is considered empty.
\subsection{Procedure Calls Instructions}
The \mintinline{nasm}{CALL} and \mintinline{nasm}{RET} instructions control the instruction flow by calling another instruction procedure\footnote{loosely synonymous with function}.
The \mintinline{nasm}{CALL} instruction takes the address of the instruction that is to be called.
Before jumping to the instruction at the given address, it \mintinline{nasm}{PUSH}es the current RIP (instruction pointer) register onto the \gls{stack}.
\mintinline{nasm}{RET} takes no operand, but instead \mintinline{nasm}{POP}s the \gls{stack}'s top entry.
The consumed value is used as a jump address.
As \mintinline{nasm}{PUSH} and \mintinline{nasm}{POP} use the RSP register, the called procedure is responsible to finish with the RSP at the same position as when it was entered.
For example, \mintinline{nasm}{PUSH}ing some value onto the stack before the end of the function would cause the \mintinline{nasm}{RET} to jump to that address instead of returning to the caller.
\subsection{Stack Frame Management Instructions}
When a procedure is called, the stack is set up with the \gls{sf}, the four components listed in \cref{lst:amd64-stack-frame-components}.
\cite[p.~48]{AMD64Vol1}:
\begin{listing}[h]
\begin{enumerate}
\item{%
Parameters passed to the called procedure (created by the calling procedure). \\
\textit{Only if parameters don't fit the \gls{cpu} registers}
}
\item{%
Return address (created by the \mintinline{nasm}{CALL} instruction). \\
\textit{Always used by \mintinline{nasm}{CALL}}
}
\item{%
Array of stack-frame pointers (pointers to stack frames of procedures with smaller nesting-level depth) which are used to access the local variables of such procedures. \\
\textit{Depends on support and implementation of nested functions in the \gls{compiler}}
}
\item{%
Local variables used by the called procedure. \\
\textit{This includes the variables passed via \gls{cpu} registers}
}
\end{enumerate}
\caption{\glsentrytext{amd64} Stack-Frame Components}
\label{lst:amd64-stack-frame-components}
\end{listing}
The \gls{amd64} manual also lists \mintinline{nasm}{ENTER} and \mintinline{nasm}{LEAVE} as instructions to \textit{"provide support for procedure calls, and are mainly used in high-level languages."}\cite[p.~48]{AMD64Vol1}.
The latter claim could not be verified by inspecting binaries produced by the \gls{C} and \gls{Rust} \glspl{compiler}.
Instead, these \glspl{compiler} generate a sequence of \mintinline{nasm}{PUSH}, \mintinline{nasm}{MOV} and \mintinline{nasm}{SUB} instructions to manage theset up the \gls{stack}.
There are instructions before and after the procedure's logic, taking care of the technicalities of \gls{stack} management.
These instruction groups within the called procedure are called prologue and epilogue.
\subsection{Full Procedure Call Example}
\label{context::os-dev-concepts::sf-handling-amd64::procedure-call-example}
This section combines the separate categories into one complete example that shows how the \gls{stack} is used by various \gls{cpu} instructions to perform procedure calls.
The following code samples are extracted from a disassembled binary which was originally created using \gls{Rust}.
The Assembler that's shown uses Intel Mnemonic, which generally operates from right to left.
For example, \mintinline{nasm}{mov a, b} copies b to a.
\cref{code::context::examples::func-callee-rust} shows the \gls{Rust} source code of the function \textit{sum}.
% \subsubsection{Top-Level Page Table Self-Reference}
% \subsubsection{Caching Lookups}
% \subsubsection{Full Example}
% * http://taptipalit.blogspot.de/2013/10/theory-recursive-mapping-page.html
% * https://www.coresecurity.com/blog/getting-physical-extreme-abuse-of-intel-based-paging-systems-part-2-windows
\begin{listing}[htb]
\tikzset{/minted/basename=callee-rust}
\begin{minted}[autogobble,linenos,breaklines=true]{rust}
TODO
\end{minted}
\caption{Procedure Call Example: Callee in Rust}
\label{code::context::examples::func-callee-rust}
\end{listing}
\begin{listing}[htb]
\tikzset{/minted/basename=callee-rust}
\begin{minted}[autogobble,linenos,breaklines=true]{nasm}
TODO
\end{minted}
\caption{Procedure Call Example: Callee in Assembly}
\label{code::context::examples::func-callee-assembly}
\end{listing}
\Cref{code::context::examples::func-caller-asm} shows a snippet of the calling function.
It stores the arguments within the registers according to the calling convention.
The caller doesn't alter the stack-frame pointer (RBP) or the stack pointer (RSP) registers before call, hence the called function must restore these if it alters them.
\begin{listing}
\begin{minted}[escapeinside=??,highlightlines={},autogobble,linenos,breaklines=true]{nasm}
TODO
\end{minted}
\caption{Procedure Call Example: Caller Assembly}
\label{code::context::examples::func-caller-asm}
\end{listing}
\begin{listing}
\begin{minted}[escapeinside=??,highlightlines={},autogobble,linenos,breaklines=true]{rust}
\end{minted}
TODO
\caption{Procedure Call Example: Caller in Rust}
\label{code::context::examples::func-caller-rust}
\end{listing}
% \balloon{comment}{
% RDI, RSI, RDX, RCX, R8, R9, XMM0–7
\begin{table}[ht!]
\centering
\begin{tabular}{ r | >{\columncolor{YellowGreen}}c | l }
\multicolumn{1}{r}{RBP offset} & \multicolumn{1}{c}{Content} & \\
$\uparrow$ & \cellcolor{white} & \\
& \cellcolor{white} \dots \textit{beyond current stack} \dots & \\
\hhline{~-~}
0 & \textit{Previous RSP} & $\leftarrow$ RBP \\
\hhline{~-~}
\vdots & \dots~~\textit{local variables}~~\dots & \\
\hhline{~-~}
-0x30 & 3rd arg & \\
\hhline{~|-|~}
-0x38 & 2nd arg & \\
\hhline{~-~}
-0x40 & 1st arg & \\
\hhline{~-~}
\vdots & \dots~~\textit{local variables}~~\dots & \\
\hhline{~-~}
-0x60 & rdi & \\
\hhline{~-~}
& \dots~~\textit{local variables}~~\dots & \\
\hhline{~-~}
$RBP-RSP$ & \textit{unknown} & $\leftarrow$ RSP \\
\hhline{~-~}
& \cellcolor{white} & \\
$\downarrow$ & \cellcolor{white} & \\
\end{tabular}
\end{table}
\cref{code::context::examples::func-prologue} shows \textit{sum}'s prologue.
The corresponding epilogue is displayed in \cref{code::context::examples::func-epilogue}.
The comments explain the code line by line, please read them to understand what exactly happens at each instruction.
\begin{listing}[ht!]
\begin{minted}[escapeinside=??,linenos=false,breaklines=true]{nasm}
$7490: push rbp ; save the stack-frame pointer on the stack
$7491: mov rbp,rsp ; set the stack-frame base pointer from the stack pointer
$7494: sub rsp,0x50 ; allocate 0x50 Bytes for arguments and local variables
$7498: mov QWORD PTR [rbp-0x30],rdi ; copy 1st arg onto stack
$749c: mov QWORD PTR [rbp-0x28],rsi ; copy 2nd arg onto stack
$74a0: mov QWORD PTR [rbp-0x20],rdx ; copy 3rd arg onto stack
\end{minted}
\caption{Function Prologue with three Arguments}
\label{code::context::examples::func-prologue}
\end{listing}
\begin{listing}[ht!]
\begin{minted}[linenos=true,breaklines=true]{nasm}
$74ee: mov rax,QWORD PTR [rbp-0x48] ; store return value in RAX
$74f2: add rsp,0x50 ; set stack pointer to where stack-frame pointer was stored
$74f6: pop rbp ; restore the stack-frame pointer
$74f7: ret ; return to the caller, following the address on the stack
\end{minted}
\caption{Function Epilogue}
\label{code::context::examples::func-epilogue}
\end{listing}
\cref{fig:proc-call-example-mem} displays
\begin{figure}
\centering
\includegraphics[width=0.95\textwidth]{gfx/call-procedure-memory-content.png}
\caption{Memory Layout Throughout The Procedure Call Steps}
\label{fig:proc-call-example-mem}
\end{figure}
\FloatBarrier
\section{Stack And Heap: Combined Usage}
\label{context::os-dev-concepts::stackheap-combined}
\Glspl{program} combine the usage of \gls{stack} and \gls{heap} use them for different purposes.
Depending on the \gls{proglang}'s \gls{compiler} and the target system, the responsibility of writing the memory management code falls either on the developer, the \gls{compiler}, or both.
@ -502,7 +782,7 @@ Depending on the \gls{proglang}'s \gls{compiler} and the target system, the resp
\toprule
\multicolumn{1}{c}{\Gls{stack}} & \multicolumn{1}{c}{Responsibility} \\
\hhline{--}
\Gls{sf} (return address, frame pointer, see \cref{rnd::sysprog-conventions::stackframe-amd64,lst:amd64-stack-frame-components}).
\Gls{sf} (return address, frame pointer, see \cref{context::os-dev-concepts::stackheap::stack}).
Procedure-local fixed-sized variables (primitive types, custom fixed-size structures, references, fixed-length arrays, etc.)
@ -537,15 +817,18 @@ Depending on the \gls{proglang}'s \gls{compiler} and the target system, the resp
\FloatBarrier
\subsection{Arrangement}
\label{context::os-dev-concepts::stackheap-combined::arrangement}
Both zones must be organized separately and arrange within the virtual address space which is assigned to process or thread.
\Cref{TODO-heap-stack-example-program} shows a \gls{C} \gls{program} and a simplified model of the hypothetical address space that would result on execution.
\Cref{fig:heap-malloc-stack-example-program} shows a \gls{C} \gls{program} and a simplified model of the hypothetical address space that would result on execution.
In this example, the \gls{stack} and \gls{heap} are placed on opposite sides of the virtual address space, and will grow towards each other.
\begin{figure}[ht!]
\includegraphics[width=\textwidth]{gfx/TODO-heap-stack-example-program}
\caption{TODO-heap-stack-example-program}
\label{TODO-heap-stack-example-program}
\centering
\includegraphics[width=0.6\textwidth]{gfx/TODO-heap-stack-example-program}
%\caption{Stack/Heap Arrangement And Dynamic Allocation \footnote{Prof. Jennifer Rexford, \url{http://slideplayer.com/slide/3288060/}}}
\label{fig:heap-malloc-stack-example-program}
% TODO: redraw
\end{figure}
% TODO: improve figure that shows stack and heap?
\FloatBarrier
The entries above \textit{"Heap"} are the different parts of the \gls{compiler} output for this program, and are loaded by the \gls{os} before the execution.
@ -555,19 +838,71 @@ RoData is read-only memory content, in this case the string literal \textit{"str
BSS contains the variable \textit{iSize}.
Lastly the \gls{stack} holds the pointer variable \textit{p}, which will reference the result of the \textit{malloc(iSize)} memory allocation.
\subsection{Safety Concerns}
\label{context::os-dev-concepts::stackheap-combined::safety-concerns}
Even though virtual address spaces are huge on \gls{amd64}, there is a slight chance that the \gls{stack} and \gls{heap} will interfere.
This could be due to direct collision, or more subtly by not detecting invalid mutual references.
\subsection{Programming Language Support}
In many \glspl{proglang} that are commonly used for \gls{app} development, the code for allocation and cleanup of \gls{heap} memory is generated by the \gls{compiler} on behalf of the programmer.
Such languages rely on the \gls{os} memory management \gls{api} and are thus not suited for developing the \gls{os} itself.
Visa-versa, languages which are suited for \gls{os} development usually don't generate \gls{heap} management code and therefore don't ensure memory-safety.
Visa-versa, languages which are suited for \gls{os} development usually don't generate \gls{heap} management code and therefore don't ensure memory-safety on the \gls{heap}.
\gls{Rust} might be an exception to this by adding static safety checks suited for \gls{os} development.
To what extend and by which means this is true, has to be confirmed or denied by the end of this work.
\gls{Rust} might be an exception to this which has to be confirmed or denied by the end of this work.
\section{Preemptive Multitasking on \glsentrytext{amd64}}
\label{context::os-dev-concepts::preemptive-multitasking-amd64}
On \gls{amd64}, the \gls{cpu}'s interrupt mechanism does not switch the full content of the context, but only handles the registers that are necessary to successfully jump to the interrupt function: RFLAGS, RSP, RBP, RIP\footnote{Segment registers are neglected}.
\subsection{The Process Context}
The content of a process's context on \gls{amd64} is given in \cref{tab:task-minimum-context-registers}.
All these registers need to be stored and restored by the \gls{os}'s interrupt handler for process preemption.
\begin{table}
\begin{tabularx}{\textwidth}{| c | X | X |}
\hline
\textbf{descriptive name} &
\textbf{register names on amd64} &
\textbf{description} \\
\hline
the instruction pointer register & RIP & address of the next instruction to be fetched \\
\hline
the stack pointer register & RSP & address of current position in stack \\
\hline
the flags register & RFLAGS & various attributes, e.g. the interrupt flag \\
\hline
all general-purpose registers & RAX, RBX, RCX, RDX, RDI, RSI, RBP, RSP, R8–R15 & arbitrary data \\
\hline
\end{tabularx}
\caption{Minimum Context Registers on amd64\cite[p.~28]{AMD64Vol2}}
\label{tab:task-minimum-context-registers}
\end{table}
\subsection{Storing The Context On The Stack}
In this scenario, the context is stored on the \gls{stack} of the function that is interrupted.
\Cref{fig:amd64-long-mode-interrupt-stac} pictures the \gls{stack} layout on interrupt entry.
In order to leverage an interrupt for a context switch, the interrupt function needs to replace these values on the \gls{stack} with values for the new context.
CS (Code-Segment) and SS (Stack-Segment) have no effect in \gls{amd64} 64-Bit mode\cite[p.~20]{AMD64Vol1} and can remain unchanged.
The \gls{os} developer needs to know the exact address where on the \gls{stack} this data structure has been pushed by the \gls{cpu}, and must then manipulate these addresses directly.
This type of manipulation is inherently dangerous and can not be easily checked by the \gls{compiler}.
The function that handles the interrupt must then use the instruction \textit{iretq}\cite[p.~252]{AMD64Vol2}, to make the \gls{cpu} restore the partial context from the \gls{stack} and continue to function pointed to by the RIP.
\begin{figure}
\centering
\includegraphics[width=0.8\textwidth]{gfx/amd64-long-mode-stack-after-interrupt.png}
\caption{Long-Mode Stack After Interrupt\cite[p.~252]{AMD64Vol2}}
\label{fig:amd64-long-mode-interrupt-stac}
\end{figure}
For a full context-switch, the other registers that are part of the context need to be handled by the \gls{os}'s interrupt function.
\chapter{Common Memory-Safety Mistakes}
\label{context::common-mem-safety-mistakes}
Building upon \cref{context::introduction}, which describes the basic mechanics of memory usage and how mistakes come to existence, this chapter presents and explains common software vulnerabilities that are related to memory-safety.
The relevant vulnerability classes are explained alongside exemplary manifestations in \gls{C}/\gls{C++}.
In \cref{rnd::porting-c-vulns}, these are ported and compared to functionally equivalent versions written in \gls{Rust}.
The previous \cref{context::introduction,context::os-dev-concepts} describe the concepts of memory management on \gls{amd64} and how mistakes might come into existence.
This chapter describes the related software weaknesses which are too commonly found.
The underlying weakness classes are explained alongside real-world and exemplary manifestations in \gls{C}.
The latter are ported and compared to functionally equivalent versions written with \gls{Rust} in \cref{rnd::porting-c-vulns},
\section{\glsentrylong{CWE}}
\label{context::common-mem-safety-mistakes::cwe}
@ -661,16 +996,14 @@ The system libraries and the kernel are written in \gls{C}, the latter containin
\gls{LX} is very popular for embedded systems, network servers and large-scale computers. % TODO: reference
Through \gls{android}, \gls{LX} has been distributed to a huge amount of mobile devices within the last decade. % TODO: reference
The list of vulnerabilities that are found in \gls{LX} device drivers which were written by \gls{android} device vendors is very concerning.
Even though Device drivers are not necessarily complex per-se, as they essentially just copy data to and from the hardware they target, but the difficulty is in performing these transfers only under safe circumstances.
\gls{LX} has a huge ecosystem with existing libraries for any imaginable use-case from cryptography to artificial intelligence to give random examples.
Even though Device drivers are not necessarily complex per-se, as they essentially just copy data to and from the hardware they target.
The difficulty is to perform these transfers only under safe circumstances, which are not always straight forward to identify or simply forgotten.
It is necessary to investigate manifestations of these errors in detail in order to analyze if these might be prevented by using \gls{Rust}.
The manifestations of memory-safety related vulnerabilities in the \gls{LX} ecosystem are given in the next section.
\gls{LX} has a huge ecosystem with existing libraries for any imaginable use-case from cryptography to artificial intelligence to give random examples.
It is necessary to investigate some of the weakness manifestations in detail in order to analyze if these might be prevented by using \gls{Rust}.
\section{Manifestations}
\label{context::common-mem-safety-mistakes::manifestations}
% Significance of the Study
% The significance is a statement of why it is important to determine the answer to the gap in the knowledge, and is related to improving the human condition. The contribution to the body of knowledge is described, and summarizes who will be able to use the knowledge to make better decisions, improve policy, advance science, or other uses of the new information. The “new” data is the information used to fill the gap in the knowledge.
This section contains real-world manifestations and \textit{re}constructed experiments of memory-safety related weaknesses.
% TODO
@ -800,10 +1133,10 @@ However, this method only verifies that the given array index is less than the m
\Cref{code::context::examples::sf-modification-simple} is a little example program in \gls{C}, which manipulates the return function address stored on the \gls{stack}.
This is done by simple and legal pointer arithmetic.
It abuses the address of the first local variable to create references into the \gls{sf} below on the \gls{stack}.
Since the first variable is on the bottom of the \gls{sf} in the function, any higher address is part of the previous \gls{sf}.
Depending on the \gls{compiler}, the return address is stored either one or two data entries below the first local variable.
In a brute-force manner the program simply overwrites both entries with a different function address.
By simply writing a different function address at these entries, the \mintinline{c}{ret} will jump there, since the original return address has been overwritten.
Since the first variable is in the beginning of the \gls{sf} of the called function, it can be used to guess the position of the return address on the \gls{stack}.
Depending on the \gls{compiler} settings, the return address is stored either one or two stack entries in front of the first local variable for a function with no arguments.
In a brute-force manner the program simply overwrites both entries with the address of \mintinline{c}{simple_printer}.
By writing a different function address at these entries, the \mintinline{c}{ret} instruction will jump there, since the original return address has been overwritten.
\begin{figure}[ht!]
\begin{subfigure}[T]{0.60\textwidth}
@ -832,7 +1165,6 @@ ret
\caption{Stack-Frame Modification}
\label{code::context::examples::sf-modification-simple}
\end{figure}
% TODO: port to rust
\Cref{TODO-callstack-manipulation} is an attempt to visualize what happens in memory and with the \gls{stack} and the \gls{cpu}'s RIP {64-Bit Instruction Pointer} register.
@ -843,9 +1175,6 @@ ret
\end{figure}
\FloatBarrier
\subsection{Uninitialized Pointers}
\begin{lstlisting}[language=C,
@ -883,7 +1212,9 @@ A recent and high impact vulnerability named \textit{Stack Clash}\footnote{https
The \gls{LX} specific vulnerability is listed as CVE-2017-1000364\footnote{http://www.cvedetails.com/cve/CVE-2017-1000364/}, where \textit{"an issue was discovered in the size of the stack guard page on Linux, specifically a 4k stack guard page is not sufficiently large and can be "jumped" over (the stack guard page is bypassed)"}.
It is assigned to the \citetitle{MITRE-CWE-119}\autocite{MITRE-CWE-119} presented in \cref{context::common-mem-safety-mistakes::cwe::119}.
\cref{context::introduction::hw-supported-mm::multilevel-paging}
\cref{context::os-dev-concepts::hw-supported-mm::multilevel-paging-concept,context::os-dev-concepts::hw-supported-mm::multilevel-paging-amd64}
\cref{context::os-dev-concepts::hw-supported-mm::page-fault}
% TODO explain that this CWE-119 vulnerability is also "Execute Code"
% TODO more references and deeper explanation of what happens: see introduction in https://www.qualys.com/2017/06/19/stack-clash/stack-clash.txt
@ -915,10 +1246,17 @@ Even if it could, testing all possible permutations of system state in every pos
The \gls{app} developer is forced to trust the underlying \gls{os}.
This puts high importance on the safety of the \gls{os} design and implementation.
\subsection{The Effects Of Programming Languages on Memory-Safety}
There are dozens of \glspl{proglang} used by humans to write \glspl{app}, but only a few are used to write \glspl{os}.
\section{Software Tests}
% TODO: describe that tests are mostly semantics as opposed to static checks being mostly syntactical and technical
% TODO: Are they necessary in addition to static checks to cover the well-known use-cases and edge-cases.
% TODO: example?
\subsubsection{Abstraction: Safety vs. Functionality}
\section{The Effects Of Programming Languages on Memory-Safety}
There are dozens of \glspl{proglang} used by humans to write \glspl{app}, but only a few are used to write \glspl{os}.
\cref{context::safe-os-dev::detecting-safety-violations-asap} explained that programming languages have direct impact on the memory-safety.
This section gives an example of how severe this impact is and explains the requirements on a \gls{os} language.
\subsection{Abstraction Tradeoffs: Safety vs. Functionality}
\label{context::introduction::memory-safety::abstr-safety-function}
In computer systems, safety and functionality are counter-proportional towards each other, because with increased functionality also grows complexity, and error cases become more difficult to find.
Applying this analogy to software development, during which the errors are created in the first place, might be misleading.
@ -929,14 +1267,9 @@ By defining an abstraction layer in form of a programming language, the language
, the language can introduce obligated rules that make the written program easier to analyze in an automated fashion, before it gets compiled into the underlying representation.
\section{Safety In Language Compilers And Static Analyzers}
\label{context::introduction::language-compilers-analyzers}
In \cref{context::introduction::memory-safety}, specifically in TODO "reference detection" was explained that programming languages have direct impact on the memory-safety.
This section gives an example of how severe this impact is and explains the requirements on a \gls{os} language.
\chapter{Mitigation Attempts}
\section{\glsentrytext{C}}
\section{Improving \glsentrytext{C}}
With the growing number of vulnerabilities, various solutions have been proposed to increase the safety of C, either with static code analysis or via \gls{compiler}-generated checks imposed at runtime. (TODO: reference).
Static analysis are not very effective on a language that has not been designed to be safety-analyzed. TODO? reference?

14
src/docs/parts/eval_and_conclusion/eval_and_conclusion.tex
View file

@ -1,18 +1,18 @@
% // vim: set ft=tex:
\chapter{Result Generalization}
\chapter{Evaluation}
\section{Low-Level Safe Abstractions in Rust}
\section{Premised Trust In Hardware}
Memory management mechanisms are partially implemented in the target system's hardware which can't be verified by at time of development.
\section{Safety Of Low-Level Code in Rust}
% TODO: Is the static analysis of hardware specific assembly code possible and useful at all?
% TODO: LLVM knows about the target and can potentially give hints about hardware specific instructions
\section{Tracking \textit{'static}ally allocated Resources}
\section{Protecting \textit{'static}ally allocated Resources}
\section{The Necessary Evils of \textit{unsafe}}
\chapter{Result Evaluation}
\paragraph{Premised Trust In Hardware}
Memory management mechanisms are partially implemented in the target system's hardware which can't be verified by at time of development.
\section{Extending Rust For Safe OS Development}
\chapter{Summary}

333
src/docs/parts/research_and_development/research_and_development.tex
View file

@ -1,20 +1,11 @@
% // vim: set ft=tex:
\chapter{Refined Research Questions}
\section{Software Tests}
% TODO: describe that tests are mostly semantics as opposed to static checks being mostly syntactical and technical
% TODO: Are they necessary in addition to static checks to cover the well-known use-cases and edge-cases.
% TODO: example?
\chapter{Concrete Research Tasks}
\section{Definition Of Additional Analysis Rules To Extend Safety Checks}
% TODO: How can Business Logical
% TODO: Business Logic Checks
% Examples:
% TLB needs to be reset on Task Change
% Registers need to be
\subsection{Paging}
Setting up and maintaining the paging-structure, as well as allocating physical memory for the virtual pages is a complex task in the \gls{os}.
Developing this part of the \gls{os} is error-prone, and is not well-supported by mainstream \glspl{proglang}.
% ISR-Stack-Frame needs to be updated on context-switch
\section{Software Fault Isolation}
% TODO: content from \cite{Balasubramanian2017}
@ -26,319 +17,17 @@ Developing this part of the \gls{os} is error-prone, and is not well-supported b
% TODO * Control access to duplicates in page tables
% TODO * Tasks can't access unallocated (physical) memory
% TODO * Tasks can't access other tasks memory
\chapter{System Programming Conventions}
\label{rnd::sysprog-conventions}
\section{Stack Frame Handling on AMD64}
\label{rnd::sysprog-conventions::stackframe-amd64}
The usage of the \gls{stack} is tightly coupled with control flow instructions in conjunction with two registers, the Stack-Frame Base Pointer (RBP) and the Stack Pointer (RSP).
The instructions that use these registers and explicitly or implicitly work with the stack\cite[p.~83]{AMD64Vol1} can be grouped into the following categories.
Together they can be used to perform \gls{stack} based procedure calls, as demonstrated in the following \cref{context::introduction::hw-supported-mm::procedure-call-example}.
\paragraph{Direct Stack Data Management} with PUSH and POP.
PUSH takes value operand which is to be pushed onto the stack.
The address in RSP moves towards numerically lower addresses with every PUSH instruction, which stores a new data entry on top.
The order is to first change the RSP and then copy the value at its new address.
POP takes a storage reference operand - \gls{cpu} register or memory address.
It works in the opposite direction to PUSH.
First, consuming the top-most data entry and storing it on the operand location, then moving the RSP address towards the numerically higher RBP address.
When RBP and RSP point to the same address, the stack is considered empty.
\paragraph{Procedure Calls} with CALL and RET. \\
These instructions control the instruction flow by calling another instruction procedure\footnote{loosely synonymous with function}.
The CALL instruction takes the address of the instruction that is to be called.
Before jumping to the instruction at the given address, it PUSHes the current RIP (instruction pointer) register onto the \gls{stack}.
RET takes no operand, but instead POPs the \gls{stack}'s top entry.
The consumed value is used as a jump address.
As PUSH and POP use the RSP register, the called procedure is responsible to finish with the RSP at the same position as when it was entered.
For example, PUSHing some value onto the stack before the end of the function would cause the RET to jump to that address instead of returning to the caller.
\paragraph{Called Procedure Setup} \emph{not} with ENTER and LEAVE.
When a procedure is called, the stack is set up with the \gls{sf}, the four components listed in \cref{lst:amd64-stack-frame-components}.
\cite[p.~48]{AMD64Vol1}:
\begin{listing}[h]
\begin{enumerate}
\item{%
Parameters passed to the called procedure (created by the calling procedure). \\
\textit{Only if parameters don't fit the \gls{cpu} registers}
}
\item{%
Return address (created by the CALL instruction). \\
\textit{Always used by CALL}
}
\item{%
Array of stack-frame pointers (pointers to stack frames of procedures with smaller nesting-level depth) which are used to access the local variables of such procedures. \\
\textit{Depends on support and implementation of nested functions in the \gls{compiler}}
}
\item{%
Local variables used by the called procedure. \\
\textit{This includes the variables passed via \gls{cpu} registers}
}
\end{enumerate}
\caption{\glsentrytext{amd64} Stack-Frame Components}
\label{lst:amd64-stack-frame-components}
\end{listing}
only necessary when there aren't enough \gls{cpu} to pass the parameters.
Item 3 is only necessary when
The \gls{amd64} manual also lists ENTER and LEAVE as instructions to \textit{"provide support for procedure calls, and are mainly used in high-level languages."}\cite[p.~48]{AMD64Vol1}.
The latter claim could not be verified by inspecting binaries produced by the \gls{C} and \gls{Rust} \glspl{compiler}.
Instead, these \glspl{compiler} generate a sequence of PUSH, MOV and SUB instructions to manage theset up the \gls{stack}.
There are instructions before and after the procedure's logic, taking care of the technicalities of \gls{stack} management.
These instruction groups within the called procedure are called prologue and epilogue.
\subsection{Full Procedure Call Example}
\label{context::introduction::hw-supported-mm::procedure-call-example}
This section combines the separate categories into one complete example that shows how the \gls{stack} is used by various \gls{cpu} instructions to perform procedure calls.
The following code samples are extracted from a disassembled binary which was originally created using \gls{Rust}.
The Assembler that's shown uses Intel Mnemonic, which generally operates from right to left.
For example, \mint{nasm}{mov a, b} copies b to a.
\cref{code::context::examples::func-callee-rust} shows the \gls{Rust} source code of the function \textit{sum}.
% \subsubsection{Top-Level Page Table Self-Reference}
% \subsubsection{Caching Lookups}
% \subsubsection{Full Example}
% * http://taptipalit.blogspot.de/2013/10/theory-recursive-mapping-page.html
% * https://www.coresecurity.com/blog/getting-physical-extreme-abuse-of-intel-based-paging-systems-part-2-windows
\begin{listing}[htb]
\tikzset{/minted/basename=callee-c}
\begin{minted}[autogobble,linenos,breaklines=true]{rust}
TODO
\end{minted}
\caption{The called function in \gls{Rust}}
\label{code::context::examples::func-callee-rust}
\end{listing}
\cref{code::context::examples::func-call-asm} shows a snippet snippet of the calling function.
It stores the arguments within the registers according to the System V X86\_64 calling convention. %TODO REFERENCE
The caller doesn't alter the stack-frame pointer (RBP) or the stack pointer (RSP) registers before call, hence the called function must restore these if it alters them.
\begin{listing}
\begin{minted}[escapeinside=??,highlightlines={},autogobble,linenos,breaklines=true]{rust}
TODO
\end{minted}
\caption{Procedure Call Example: Caller Rust}
\label{code::context::examples::func-call-asm}
\end{listing}
\begin{listing}
\begin{minted}[escapeinside=??,highlightlines={},autogobble,linenos,breaklines=true]{nasm}
\end{minted}
TODO
\caption{Procedure Call Example: Caller Assembly}
\label{code::context::examples::func-call-rust}
\end{listing}
% \balloon{comment}{
% RDI, RSI, RDX, RCX, R8, R9, XMM0–7
\begin{table}[ht!]
\tikzmark{precallto}
\centering
\begin{tabular}{ r | >{\columncolor{YellowGreen}}c | l }
\multicolumn{1}{r}{RBP offset} & \multicolumn{1}{c}{Content} & \\
$\uparrow$ & \cellcolor{white} & \\
& \cellcolor{white} \dots \textit{beyond current stack} \dots & \\
\hhline{~-~}
0 & \textit{Previous RSP} & $\leftarrow$ RBP \\
\hhline{~-~}
\vdots & \dots~~\textit{local variables}~~\dots & \\
\hhline{~-~}
-0x30 & 3rd arg & \\
\hhline{~|-|~}
-0x38 & 2nd arg & \\
\hhline{~-~}
-0x40 & 1st arg & \\
\hhline{~-~}
\vdots & \dots~~\textit{local variables}~~\dots & \\
\hhline{~-~}
-0x60 & rdi & \\
\hhline{~-~}
& \dots~~\textit{local variables}~~\dots & \\
\hhline{~-~}
$RBP-RSP$ & \textit{unknown} & $\leftarrow$ RSP \\
\hhline{~-~}
& \cellcolor{white} & \\
$\downarrow$ & \cellcolor{white} & \\
\end{tabular}
\end{table}
\cref{code::context::examples::func-prologue} shows \textit{sum}'s prologue.
The corresponding epilogue is displayed in \cref{code::context::examples::func-epilogue}.
The comments explain the code line by line, please read them to understand what exactly happens at each instruction.
\begin{listing}[ht!]
\begin{minted}[escapeinside=??,linenos=false,breaklines=true]{nasm}
$7490: push ?\tikzmark{prologuestart}? rbp ; save the stack-frame pointer on the stack
$7491: mov rbp,rsp ; set the stack-frame base pointer from the stack pointer
$7494: sub rsp,0x50 ; allocate 0x50 Bytes for arguments and local variables
$7498: mov QWORD PTR [rbp-0x30],rdi ; copy 1st arg onto stack
$749c: mov QWORD PTR [rbp-0x28],rsi ; copy 2nd arg onto stack
$74a0: mov QWORD PTR [rbp-0x20],rdx ; copy 3rd arg onto stack
\end{minted}
\caption{Function Prologue with three Arguments}
\label{code::context::examples::func-prologue}
\end{listing}
\begin{tikzpicture}[remember picture]
\draw[overlay,red,thick,dashed] (pic cs:precallto) circle [radius=7pt] node { \textbf{1} };
\draw[overlay,red,thick,dashed] (pic cs:prologuestart) circle [radius=7pt] node { \textbf{1} };
\end{tikzpicture}
\begin{listing}[ht!]
\begin{minted}[linenos=true,breaklines=true]{nasm}
$74ee: mov rax,QWORD PTR [rbp-0x48] ; store return value in RAX
$74f2: add rsp,0x50 ; set stack pointer to where stack-frame pointer was stored
$74f6: pop rbp ; restore the stack-frame pointer
$74f7: ret ; return to the caller, following the address on the stack
\end{minted}
\caption{Function Epilogue}
\label{code::context::examples::func-epilogue}
\end{listing}
\cref{fig:proc-call-example-mem} displays
\begin{figure}
\centering
\includegraphics[width=0.95\textwidth,]{gfx/call-procedure-memory-content.png}
\caption{Memory Layout Throughout The Procedure Call Steps}
\label{fig:proc-call-example-mem}
\end{figure}
\FloatBarrier
\section{4-Level Paging Hierarchy on \glsentrytext{amd64}}
\label{rnd::sysprog-conventions::paging-amd64}
On \gls{amd64} "a four-level page-translation data structure is provided to allow long-mode operating systems to translate a 64-Bit virtual-address space into a 52-Bit physical-address space."\cite[p.~18]{AMD64Vol2}.
This allows the system to only hold the \textit{PML4} table, the which is currently referenced by the \textit{Page Map Base Register (CR3)}, available in main memory.
\cref{fig:virtual-addr-transl} shows the 64-Bit virtual address composition on \gls{amd64}, which uses four-levels of page tables.
Counterintuitively the page-tables are not called level-\textit{n}-page-table, but the levels received distinct names in \citetitle{AMD64Vol2}.
The most-significant Bits labelled as \textit{Sign Extend} are not used for addressing purposes, but must adhere the canonical address form and simply repeat the value of the most-significant implemented Bit \cite[p.~130]{AMD64Vol2}.
The least significant Bits represent the offset within the physical page.
The four groups in between are used to index the page-table at their respective level.
\begin{figure}
\centering
\includegraphics[width=\textwidth]{gfx/Virtual-to-Physical-Address-Translation-Long-Mode.png}
\caption{Virtual to Physical Address in Long Mode\cite{AMD64Vol2}}
\label{fig:virtual-addr-transl}
\end{figure}
\subsubsection{Translation Scheme 4 KiB and 2 MiB Pages}
The \gls{amd64} architecture allows configuring the page-size, two of which will be introduced in this section.
\cref{tab:page-transl-vaddr-composition} displays the virtual address composition for the 4KiB and 2MiB page-size modes on \gls{amd64}.
The direction from top to bottom in the table corresponds to most significant to least significant - left to right - in the virtual address.
The \textit{sign extension} Bits cannot be used for actual information but act as a reservation for future architectural changes.
\begin{table}
\begin{tabular}{l | c | c}
Description & Bits in 4 KiB Pages & Bits in 2 MiB Pages \\
\hline
Sign Extend & 12 & 12 \\
Page-Map-Level-4 Offeset & 9 & 9 \\
Page-Directory-Pointer Offeset & 9 & 9 \\
Page-Directory Offeset & 9 & 9 \\
Page-Table Offeset & 9 & - \\
Physical Page Offset & 9 & 21 \\
\end{tabular}
\caption{Paging on \gls{amd64}: Virtual Address Composition 4KiB/2MiB pagesizes}
\label{tab:page-transl-vaddr-composition}
\end{table}
\begin{figure}
\centering
\includegraphics[width=\textwidth]{gfx/amd64-4kb-page-translation-long-mode}
\caption{4-Kbyte Page Translation—Long Mode\cite{AMD64Vol2}}
\label{fig:4kb-page-transl}
\end{figure}
\cref{fig:4kb-page-transl} shows the detailed virtual address composition for 4 KiB pages, using four levels of page-tables.
It uses four sets of 9-Bit indices in the virtual address, one per hierarchy level, followed by the 9 Bit page-internal offset.
An alternative approach is displayed in \cref{fig:2mb-page-transl}, using 2 MiB sized pages.
It uses three sets of 9-Bit indices for the page-tables, and a 21-Bit page-internal offset.
Increasing the page-size improves speed and memory-usage and decreases the granularity.
In this specific example the hierarchy is reduced by one level of page-tables.
This reduces the amount of storage required for the page-tables in overall and causes the lookup algorithm to finish faster.
\begin{figure}
\centering
\includegraphics[width=\textwidth]{gfx/amd64-2mb-page-translation-long-mode}
\caption{2-Mbyte Page Translation—Long Mode\cite{AMD64Vol2}}
\label{fig:2mb-page-transl}
\end{figure}
The other supported page sizes, 4 MiB and 1 GiB, as well as intermixing page sizes through the different levels don't add new insight into the mechanism and don't need to be detailed here.
\section{Interrupt Driven Preemptive Context Switches on \glsentrytext{amd64}}
\label{rnd::sysprog-conventions::ir-driven-preemptive-cs-amd64}
On \gls{amd64}, the \gls{cpu}'s interrupt mechanism does not switch the full context described previously, but only handles the registers that are necessary to successfully jump to the interrupt function: RFLAGS, RSP, RBP, RIP\footnote{Segment registers are neglected}.
\subsection{Interrupts}
% TODO https://software.intel.com/sites/default/files/managed/39/c5/325462-sdm-vol-1-2abcd-3abcd.pdf p. 2848
\subsection{Context Content}
A description for \gls{amd64} is given in \cref{tab:task-minimum-context-registers}.
\begin{table}
\begin{tabularx}{\textwidth}{| c | X | X |}
\hline
\textbf{descriptive name} &
\textbf{register names on amd64} &
\textbf{description} \\
\hline
the instruction pointer register & RIP & address of the next instruction to be fetched \\
\hline
the stack pointer register & RSP & address of current position in stack \\
\hline
the flags register & RFLAGS & various attributes, e.g. the interrupt flag \\
\hline
all general-purpose registers & RAX, RBX, RCX, RDX, RDI, RSI, RBP, RSP, R8–R15 & arbitrary data \\
\hline
\end{tabularx}
\caption{Minimum Context Registers on amd64\cite[p.~28]{AMD64Vol2}}
\label{tab:task-minimum-context-registers}
\end{table}
\subsection{Storing The Context On The Stack}
In this scenario, the context is stored on the \gls{stack} of the function that is interrupted.
\Cref{fig:amd64-long-mode-interrupt-stac} pictures the \gls{stack} layout on interrupt entry.
In order to leverage an interrupt for a context switch, the interrupt function needs to replace these values on the \gls{stack} with values for the new context.
CS (Code-Segment) and SS (Stack-Segment) have no effect in \gls{amd64} 64-Bit mode\cite[p.~20]{AMD64Vol1} and can remain unchanged.
The \gls{os} developer needs to know the exact address where on the \gls{stack} this data structure has been pushed by the \gls{cpu}, and must then manipulate these addresses directly.
This type of manipulation is inherently dangerous and can not be easily checked by the \gls{compiler}.
The function that handles the interrupt must then use the instruction \textit{iretq}\cite[p.~252]{AMD64Vol2}, to make the \gls{cpu} restore the partial context from the \gls{stack} and continue to function pointed to by the RIP.
\begin{figure}
\centering
\includegraphics[width=0.8\textwidth]{gfx/amd64-long-mode-stack-after-interrupt.png}
\caption{Long-Mode Stack After Interrupt\cite[p.~252]{AMD64Vol2}}
\label{fig:amd64-long-mode-interrupt-stac}
\end{figure}
For a full context-switch, the other registers that are part of the context need to be handled by the \gls{os}'s interrupt function.
\subsection{Paging}
Setting up and maintaining the paging-structure, as well as allocating physical memory for the virtual pages is a complex task in the \gls{os}.
Developing this part of the \gls{os} is error-prone, and is not well-supported by mainstream \glspl{proglang}.
\chapter{Porting \glsentrytext{C} Vulnerabilities}
\label{rnd::porting-c-vulns}
In this chapter, the weakness manifestations from \cref{context::common-mem-safety-mistakes::manifestations} are rewritten in \gls{Rust} to learn to what level they are mitigated just by porting them.
In this chapter, the weakness manifestations given in \cref{context::common-mem-safety-mistakes::manifestations} are rewritten in \gls{Rust} to examine if these are mitigated just by porting them.
This is done incrementally by first porting the vulnerability to unsafe Rust, followed by a rewrite to drop all unsafe code but keeping the intended functionality.
% TODO stack frame manipulation
% TODO official CWE-119 examples
\chapter{\glsentrytext{LX} Modules Written In \glsentrytext{Rust}}

42
src/docs/thesis.bib
View file

@ -3,10 +3,15 @@ Any changes to this file will be lost if it is regenerated by Mendeley.
BibTeX export options can be customized via Options -> BibTeX in Mendeley Desktop
@article{Junker,
author = {Junker, Stefan},
file = {:home/steveej/src/steveej/msc-thesis/src/docs/thesis.pdf:pdf},
title = {{Guarantees On In-Kernel Memory-Safety Using Rust's Static Code Analysis}}
@article{Matz2009,
author = {Matz, M and Hubicka, J and Jaeger, a and Mitchell, M},
file = {:home/steveej/src/steveej/msc-thesis/docs/System V Application Binary Interface AMD64 Architecture Processor Supplement Draft Version 0.99.7.pdf:pdf},
isbn = {013877630X},
pages = {1--128},
pmid = {2477614},
title = {{System V Application Binary Interface AMD64 Architecture Processor Supplement}},
url = {papers2://publication/uuid/CD8D5668-B1F5-4FE3-BAD8-25F1E589A9E5},
year = {2009}
}
@article{Lattner2005,
abstract = {The LLVM Compiler Infrastructure (http://llvm.cs. uiuc.edu) is a$\backslash$nrobust system that is well suited for a wide variety of research$\backslash$nand development work. This brief paper introduces the LLVM system$\backslash$nand provides pointers to more extensive documentation, complementing$\backslash$nthe tutorial presented at LCPC.},
@ -74,17 +79,6 @@ title = {{From Collision To Exploitation: Unleashing Use-After-Free Vulnerabilit
url = {http://dl.acm.org/citation.cfm?doid=2810103.2813637},
year = {2015}
}
@article{Merity2016,
abstract = {Recent neural network sequence models with softmax classifiers have achieved their best language modeling performance only with very large hidden states and large vocabularies. Even then they struggle to predict rare or unseen words even if the context makes the prediction unambiguous. We introduce the pointer sentinel mixture architecture for neural sequence models which has the ability to either reproduce a word from the recent context or produce a word from a standard softmax classifier. Our pointer sentinel-LSTM model achieves state of the art language modeling performance on the Penn Treebank (70.9 perplexity) while using far fewer parameters than a standard softmax LSTM. In order to evaluate how well language models can exploit longer contexts and deal with more realistic vocabularies and larger corpora we also introduce the freely available WikiText corpus.},
archivePrefix = {arXiv},
arxivId = {1609.07843},
author = {Merity, Stephen and Xiong, Caiming and Bradbury, James and Socher, Richard},
eprint = {1609.07843},
journal = {Arxiv},
title = {{Pointer Sentinel Mixture Models}},
url = {http://arxiv.org/abs/1609.07843},
year = {2016}
}
@inproceedings{Ma2013,
abstract = {—Aiming at the problem of higher memory consumption and lower execution efficiency during the dynamic detecting to C/C++ programs memory vulnerabilities, this paper presents a dynamic detection method called ISC. The ISC improves the Safe-C using pointer analysis technology. Firstly, the ISC defines a simple and efficient fat pointer representation instead of the safe pointer in the Safe-C. Furthermore, the ISC uses the unification-based analysis algorithm with one level flow static pointer. This identification reduces the number of pointers that need to be converted to fat pointers. Then in the process of program running, the ISC detects memory vulnerabilities through constantly inspecting the attributes of fat pointers. Experimental results indicate that the ISC could detect memory vulnerabilities such as buffer overflows and dangling pointers. Comparing with the Safe-C, the ISC dramatically reduces the memory consumption and lightly improves the execution efficiency.},
author = {Ma, Rui and Chen, Lingkui and Hu, Changzhen and Xue, Jingfeng and Zhao, Xiaolin},
@ -97,6 +91,19 @@ pages = {52--57},
title = {{A dynamic detection method to C/C++ programs memory vulnerabilities based on pointer analysis}},
year = {2013}
}
@article{Mailloux1969,
author = {Mailloux, B. J. and Peck, J. E L and Koster, C. H A},
doi = {10.1007/BF02163002},
file = {:home/steveej/src/steveej/msc-thesis/docs/Algol68-RevisedReport.pdf:pdf},
isbn = {978-3-662-38646-0},
issn = {0029599X},
journal = {Numerische Mathematik},
number = {2},
pages = {79--218},
title = {{Report on the Algorithmic Language ALGOL 68}},
volume = {14},
year = {1969}
}
@article{Corporation2011,
abstract = {The Intel{\{}$\backslash$textregistered{\}} 64 and IA-32 Architectures Software Developer's Manual, Volume 1, describes the basic architecture and programming environment of Intel 64 and IA-32 processors. The Intel{\{}$\backslash$textregistered{\}} 64 and IA-32 Architectures Software Developer's Manual, Volumes 2A {\&} 2B, describe the instruction set of the processor and the opcode struc- ture. These volumes apply to application programmers and to programmers who write operating systems or executives. The Intel{\{}$\backslash$textregistered{\}} 64 and IA-32 Architectures Software Developer's Manual, Volumes 3A {\&} 3B, describe the operating-system support environment of Intel 64 and IA-32 processors. These volumes target operating- system and BIOS designers. In addition, the Intel{\{}$\backslash$textregistered{\}} 64 and IA-32 Architectures Software Developer's Manual, Volume 3B, addresses the programming environment for classes of software that host operating systems.},
author = {Corporation, Intel},
@ -191,6 +198,11 @@ title = {{AMD64 Architecture Programmer's Manual Volume 2: System Programming}},
volume = {1},
year = {2012}
}
@article{Junker,
author = {Junker, Stefan},
file = {:home/steveej/src/steveej/msc-thesis/src/docs/thesis.pdf:pdf},
title = {{Guarantees On In-Kernel Memory-Safety Using Rust's Static Code Analysis}}
}
@article{Nilsson2017,
author = {Nilsson, Fredrik},
file = {:home/steveej/src/github/steveej/msc-thesis/docs/A Rust-based Runtime for the Internet of Things.pdf:pdf},

3
src/docs/thesis.tex
View file

@ -50,6 +50,7 @@
\appto\theFancyVerbLine{\tikzmark{\pgfkeysvalueof{/minted/basename}\arabic{FancyVerbLine}}}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{wrapfig}
@ -71,7 +72,7 @@
\newcommand{\buzzwords}{memory-safety, operating system development, rust, static software analysis, software vulnerability}
% Numbered Subsubsections
\setcounter{secnumdepth}{5}
\setcounter{secnumdepth}{3}
\date{Summersemester 2017}
\title{\topic}