Commit 542c121d authored by Christian Jürges's avatar Christian Jürges
Browse files

Initial commit

parents
Pipeline #418 failed with stages
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="PublishConfigData" autoUpload="Always" serverName="vm.eqipe.cloud">
<serverData>
<paths name="vm.eqipe.cloud">
<serverdata>
<mappings>
<mapping deploy="/" local="$PROJECT_DIR$" web="/" />
</mappings>
</serverdata>
</paths>
</serverData>
<option name="myAutoUpload" value="ALWAYS" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding" addBOMForNewFiles="with NO BOM" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/zfsumounter.iml" filepath="$PROJECT_DIR$/.idea/zfsumounter.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="WebServers">
<option name="servers">
<webServer id="c4f97fc4-65e5-4108-938a-b12a3b5a59c6" name="vm.eqipe.cloud" url="http://vm.eqipe.cloud">
<fileTransfer host="vm.eqipe.cloud" port="22" privateKey="$USER_HOME$/.ssh/id_rsa" rootFolder="/opt/zfsumounter" accessType="SFTP" keyPair="true">
<advancedOptions>
<advancedOptions dataProtectionLevel="Private" passiveMode="true" shareSSLContext="true" />
</advancedOptions>
<option name="port" value="22" />
</fileTransfer>
</webServer>
</option>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="085b6e34-2887-457c-96ef-ca9c937a2df8" name="Default Changelist" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/umounter.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="765">
<caret line="51" column="5" selection-start-line="49" selection-start-column="4" selection-end-line="51" selection-end-column="5" />
<folding>
<element signature="e#14#148#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/localfs.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1650">
<caret line="110" column="48" selection-start-line="110" selection-start-column="48" selection-end-line="110" selection-end-column="48" />
<folding>
<element signature="e#14#74#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/nfsfs.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180">
<caret line="12" column="26" selection-start-line="12" selection-start-column="26" selection-end-line="12" selection-end-column="26" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/utils.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="765">
<caret line="60" column="27" selection-start-line="60" selection-start-column="27" selection-end-line="60" selection-end-column="27" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/error.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150">
<caret line="12" column="16" selection-start-line="12" selection-start-column="16" selection-end-line="12" selection-end-column="16" />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="Go File" />
</list>
</option>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>umountByLocalPath</find>
<find>localNFSMountMap</find>
</findStrings>
</component>
<component name="GOROOT" path="/usr/local/Cellar/go/1.11.2/libexec" />
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/error.go" />
<option value="$PROJECT_DIR$/utils.go" />
<option value="$PROJECT_DIR$/nfsfs.go" />
<option value="$PROJECT_DIR$/localfs.go" />
<option value="$PROJECT_DIR$/umounter.go" />
</list>
</option>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="1527" />
<option name="y" value="-486" />
<option name="width" value="1661" />
<option name="height" value="1054" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="zfsumounter" type="b2602c69:ProjectViewProjectNode" />
<item name="zfsumounter" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="DefaultGoTemplateProperty" value="Go File" />
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="go.gopath.indexing.explicitly.defined" value="true" />
<property name="last_opened_file_path" value="$USER_HOME$" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="$PROJECT_DIR$" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="ToolWindowManager">
<frame x="1527" y="-486" width="1661" height="1054" extended-state="0" />
<editor active="true" />
<layout>
<window_info content_ui="combo" id="Project" order="0" visible="true" weight="0.22112416" />
<window_info id="Structure" order="1" side_tool="true" weight="0.25" />
<window_info id="Favorites" order="2" side_tool="true" />
<window_info anchor="bottom" id="Message" order="0" />
<window_info anchor="bottom" id="Find" order="1" weight="0.32949087" />
<window_info anchor="bottom" id="Run" order="2" />
<window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="4" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="6" />
<window_info anchor="bottom" id="Docker" order="7" />
<window_info anchor="bottom" id="Version Control" order="8" />
<window_info active="true" anchor="bottom" id="File Transfer" order="9" visible="true" weight="0.32848233" />
<window_info anchor="bottom" id="Database Changes" order="10" />
<window_info anchor="bottom" id="Terminal" order="11" />
<window_info anchor="bottom" id="Event Log" order="12" side_tool="true" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="0" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="Ant Build" order="1" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
<window_info anchor="right" id="Database" order="3" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="editorHistoryManager">
<entry file="file:///usr/local/Cellar/go/1.11.2/libexec/src/os/exec/exec.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="338">
<caret line="304" column="14" selection-start-line="304" selection-start-column="14" selection-end-line="304" selection-end-column="14" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/utils.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="765">
<caret line="60" column="27" selection-start-line="60" selection-start-column="27" selection-end-line="60" selection-end-column="27" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/error.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="150">
<caret line="12" column="16" selection-start-line="12" selection-start-column="16" selection-end-line="12" selection-end-column="16" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/localfs.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1650">
<caret line="110" column="48" selection-start-line="110" selection-start-column="48" selection-end-line="110" selection-end-column="48" />
<folding>
<element signature="e#14#74#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/umounter.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="765">
<caret line="51" column="5" selection-start-line="49" selection-start-column="4" selection-end-line="51" selection-end-column="5" />
<folding>
<element signature="e#14#148#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/nfsfs.go">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180">
<caret line="12" column="26" selection-start-line="12" selection-start-column="26" selection-end-line="12" selection-end-column="26" />
</state>
</provider>
</entry>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
package main
import (
"fmt"
)
// Error is an error which is returned when the `zfs` or `zpool` shell
// commands return with a non-zero exit code.
type Error struct {
Err error
Debug string
Stderr string
ExitCode int
}
// Error returns the string representation of an Error.
func (e Error) Error() string {
return fmt.Sprintf("%s: %q => %s", e.Err, e.Debug, e.Stderr)
}
package main
import (
"bufio"
"io/ioutil"
"os"
"strconv"
"strings"
)
var localNFSMountMap = make(map[string]*mnts)
// nfs server mount point
type nfsMountPoint struct {
server string
serverPath string
localPath string
dtype string
opts []string
freq int
pass int
}
// local nfs mounts registered in /proc/mounts
type mnts struct {
version uint64
device string
localPath string
isStale bool
isDeleted bool
unmountFailCount uint8
}
// verify nfs exported mounts
func verifyNFSExportedMounts() {
// log.Debug("poll nr %v", pollCounter)
// test if we got old not existing mounts
// check for stales for currently exported
for k, v := range nfsExportMap {
log.Debug(k, v.localPath)
nfsExportMap[k].isStale = isPathIsStale(v.localPath)
if nfsExportMap[k].unmountFailCount < 10 {
if nfsExportMap[k].isStale {
err := umountByLocalPath(v.localPath)
if err != nil {
nfsExportMap[k].unmountFailCount++
} else {
nfsExportMap[k].isDeleted = true
}
} else {
// not stale any longer? Reset counter
nfsExportMap[k].unmountFailCount = 0
}
}
}
}
// verify if local nfs mounts are still healthy
func verifyLocalNFSMounts() {
for k, v := range localNFSMountMap {
log.Debug(k, v.localPath)
localNFSMountMap[k].isStale = isPathIsStale(v.localPath)
if localNFSMountMap[k].isStale {
// try to umount
err := umountByLocalPath(v.localPath)
if err != nil {
localNFSMountMap[k].unmountFailCount++
} else {
localNFSMountMap[k].isDeleted = true
}
} else {
localNFSMountMap[k].unmountFailCount = 0
}
}
}
// read linux /proc/mounts and
// extract nfs or nfs4 to MountPoints slice struct
func getMountsLinux() (string, error) {
file, err := os.Open("/proc/mounts")
if err != nil {
log.Error(err)
return "", err
}
// defer file.Close()
sc := bufio.NewScanner(file)
// mountPoints = nil
// var mountPoints = make(map[string]*nfsMountPoint)
// mountPoints = make(map[string]*nfsMountPoint)
localNFSMountMap = make(map[string]*mnts)
for sc.Scan() {
s := strings.Split(sc.Text(), " ")
if len(s) == 6 {
// only track nfs shares
if s[2] == "nfs" || s[2] == "nfs4" {
// log.Info(sc.Text())
var mp nfsMountPoint
serverAddressPath := strings.Split(s[0], ":")
if len(serverAddressPath) == 2 {
mp.server = serverAddressPath[0]
mp.serverPath = serverAddressPath[1]
mp.localPath = s[1]
if strings.HasPrefix(mp.localPath, opts.LocalBasePath) {
mp.dtype = s[2]
opts := strings.Split(s[3], ",")
mp.opts = append(mp.opts, opts...)
mp.freq, _ = strconv.Atoi(s[4])
mp.pass, _ = strconv.Atoi(s[5])
var sp = mp.server + ":" + mp.serverPath
md5ServerPath := GetMD5Hash(sp)
// mountPoints[md5ServerPath] = &mp
// log.Info("full server path:", sp, md5ServerPath)
_, ok := localNFSMountMap[md5ServerPath]
if ok {
localNFSMountMap[md5ServerPath].version = pollCounter
localNFSMountMap[md5ServerPath].isDeleted = false
} else {
log.Debug("Adding localNFSMountMap with hash %s (%s) --> %s ", md5ServerPath, sp, mp.localPath)
localNFSMountMap[md5ServerPath] = &mnts{version: pollCounter, device: sp, localPath: mp.localPath}
}
}
} else {
log.Warning("Odd nfs server address:", s[0])
}
}
}
}
file.Close()
// if len(localNFSMountMap) > len(mountPoints) {
// log.Info(mountPoints)
/*
for k, v := range localNFSMountMap {
_, ok := mountPoints[k]
if !ok {
log.Infof("mount %v (%s) no longer exists --> removing from map", k, v.device)
delete(localNFSMountMap, k)
}
} */
// }
if scanErr := sc.Err(); scanErr != nil {
log.Fatal(scanErr)
}
// log.Printf("mps: %#v", mountPoints)
// log.Printf("mps: %#v", localNFSMountMap)
return "", err
}
// check if local directory is readable
func isPathIsStale(localPath string) bool {
log.Debug("trying to read localPath", localPath)
_, err := ioutil.ReadDir(localPath)
var isStale = false
if err != nil {
log.Error(err)
// isStale = strings.Contains(err.Error(), "stale")
isStale = true
}
return isStale
}
// umoumt by local path
func umountByLocalPath(localPath string) error {
log.Info("trying to umount", localPath)
c := command{Command: "/bin/umount"}
args := make([]string, 3)
args[0] = "-v"
args[1] = "-l"
args[2] = localPath
_, err := c.Run(args...)
if err != nil {
log.Error("error calling showmount", err)
}
return err
}
package main
import "strings"
type nfsExport struct {
Server string
Path string
Options string
}
// var nfsExportMap = make(map[string]*mnts)
var nfsExportMap map[string]*mnts
var nfsExports []nfsExport
// get a list of exports by
// using /sbin/showmount
func getNFSExports() {
c := command{Command: "/sbin/showmount"}
// var args = []string{"--no-headers", "--exports", "192.168.99.9"}
args := make([]string, 3)
args[0] = "--no-headers"
args[1] = "--exports"
args[2] = opts.NfsServerAddress
// log.Printf("args: %#v", args)
out, err := c.Run(args...)
if err != nil {
log.Error("error calling showmount: %v", err)
return
}
// log.Printf("showmount: %v", out)
nfsExports = nil
nfsExportMap = make(map[string]*mnts)
// parse
for _, l := range out {
// log.Info(l)
// p := strings.Split(l, " ")
// log.Printf("%v %#v", i, l)
// log.Printf("%v %#v", i, l)
var n nfsExport
n.Server = opts.NfsServerAddress
n.Path = l[0]
// only add wanted directories
if strings.HasPrefix(n.Path, fullZfsServerPath) {
// log.Debug("adding", n.Path)
localPath := createLocalPath(n.Path)
n.Options = l[1]
nfsExports = append(nfsExports, n)
var sp = n.Server + ":" + n.Path
md5ServerPath := GetMD5Hash(sp)
_, ok := nfsExportMap[md5ServerPath]
if ok {
nfsExportMap[md5ServerPath].version = pollCounter
nfsExportMap[md5ServerPath].isDeleted = false
} else {
nfsExportMap[md5ServerPath] = &mnts{version: pollCounter, device: sp, localPath: localPath}
}
}
}
// log.Printf("nfs exports: %#v", nfsExports)
// log.Printf("nfs exports: %#v", nfsExportMap)
}
package main
import (
"encoding/json"
"fmt"
"github.com/jessevdk/go-flags"
"github.com/op/go-logging"
"net"
"os"
"strings"
"sync"
"time"
)
var pollCounter uint64
var fullZfsServerPath = ""
var log = logging.MustGetLogger("umounter")
var logFormat = logging.MustStringFormatter(
`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
)
// struct for receiving messages from
// zfs rest server
type NotifyStruct struct {
Type string `json:"type"`
Content string `json:"content"`
}
// struct of options
var opts struct {
NfsServerAddress string `long:"nfsserver" description:"NFS server address (example --nfsserver=192.168.99.9)" required:"false" default:"10.0.3.9"`
LocalBasePath string `long:"localbasepath" description:"root path" required:"false" default:"/home/xcloud/data_quota"`
ZfsRestServerAddress string `long:"zfsrestserver" description:"ZFS REST server address:Port (example --zfsrestserver=192.168.99.9:8000" required:"false" default:"10.0.3.9:6000"`
ZfsServerBasePath string `long:"zfsserverbasepath" description:"zfs base pool and path (no leading slash allowed!)" required:"false" default:"pool01/ocnc/data_quota"` // no leading slash!
LogLevel string `long:"loglevel" description:"set log level" required:"false" default:"info"`
}
// main
func main() {
_, err := flags.Parse(&opts)
// validate params
if strings.HasPrefix(opts.ZfsServerBasePath, "/") {
fmt.Printf("param zfsserverbasepath has leading slash! Please remove it!")
os.Exit(-1)
}
fullZfsServerPath = "/" + opts.ZfsServerBasePath
if err != nil {
os.Exit(1)
}
logLevel, err := logging.LogLevel(opts.LogLevel)
if err != nil {
panic(err)