mirror of
https://github.com/OpenMaxIO/openmaxio-object-browser
synced 2026-07-01 07:41:18 -07:00
Add admin heal api and ui (#142)
This commit is contained in:
1
go.sum
1
go.sum
@@ -406,6 +406,7 @@ github.com/minio/lsync v1.0.1 h1:AVvILxA976xc27hstd1oR+X9PQG0sPSom1MNb1ImfUs=
|
||||
github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA=
|
||||
github.com/minio/mc v0.0.0-20200515235434-3b479cf92ed6 h1:2SrKe2vLDLwvnYkYrJelrzyGW8t/8HCbr9yDsw+8XSI=
|
||||
github.com/minio/mc v0.0.0-20200515235434-3b479cf92ed6/go.mod h1:U3Jgk0bcSjn+QPUMisrS6nxCWOoQ6rYWSvLCB30apuU=
|
||||
github.com/minio/mc v0.0.0-20200519213124-bf731558cda0 h1:D497vXgHka7/Z3oYFEbFIfAEdWBHbfad8KZ5grSROI0=
|
||||
github.com/minio/minio v0.0.0-20200421050159-282c9f790a03/go.mod h1:zBua5AiljGs1Irdl2XEyiJjvZVCVDIG8gjozzRBcVlw=
|
||||
github.com/minio/minio v0.0.0-20200516011754-9cac385aecdb h1:CQC7D3UDnUycuxhwImcVhMSLet/RbShosAnYcvMtEB8=
|
||||
github.com/minio/minio v0.0.0-20200516011754-9cac385aecdb/go.mod h1:wymaytM/HELuwdz7BGZHmQ3XKq2SxPsLeGxyOCaCLiA=
|
||||
|
||||
@@ -35,6 +35,7 @@ var (
|
||||
serviceAccounts = "/service-accounts"
|
||||
clusters = "/clusters"
|
||||
clustersDetail = "/clusters/:clusterName"
|
||||
heal = "/heal"
|
||||
)
|
||||
|
||||
type ConfigurationActionSet struct {
|
||||
@@ -195,6 +196,16 @@ var clustersActionSet = ConfigurationActionSet{
|
||||
actions: iampolicy.NewActionSet(),
|
||||
}
|
||||
|
||||
// healActionSet contains the list of admin actions required for this endpoint to work
|
||||
var healActionSet = ConfigurationActionSet{
|
||||
actionTypes: iampolicy.NewActionSet(
|
||||
iampolicy.AllAdminActions,
|
||||
),
|
||||
actions: iampolicy.NewActionSet(
|
||||
iampolicy.HealAdminAction,
|
||||
),
|
||||
}
|
||||
|
||||
// endpointRules contains the mapping between endpoints and ActionSets, additional rules can be added here
|
||||
var endpointRules = map[string]ConfigurationActionSet{
|
||||
configuration: configurationActionSet,
|
||||
@@ -212,6 +223,7 @@ var endpointRules = map[string]ConfigurationActionSet{
|
||||
serviceAccounts: serviceAccountsActionSet,
|
||||
clusters: clustersActionSet,
|
||||
clustersDetail: clustersActionSet,
|
||||
heal: healActionSet,
|
||||
}
|
||||
|
||||
// GetActionsStringFromPolicy extract the admin/s3 actions from a given policy and return them in []string format
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestGetAuthorizedEndpoints(t *testing.T) {
|
||||
"admin:*",
|
||||
},
|
||||
},
|
||||
want: 12,
|
||||
want: 13,
|
||||
},
|
||||
{
|
||||
name: "all s3 endpoints",
|
||||
@@ -78,7 +78,7 @@ func TestGetAuthorizedEndpoints(t *testing.T) {
|
||||
"s3:*",
|
||||
},
|
||||
},
|
||||
want: 15,
|
||||
want: 16,
|
||||
},
|
||||
{
|
||||
name: "no endpoints",
|
||||
|
||||
File diff suppressed because one or more lines are too long
5976
portal-ui/package-lock.json
generated
5976
portal-ui/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,7 @@
|
||||
"@types/webpack-env": "^1.14.1",
|
||||
"@types/websocket": "^1.0.0",
|
||||
"ansi-to-react": "^6.0.5",
|
||||
"chart.js": "^2.9.3",
|
||||
"codemirror": "^5.52.2",
|
||||
"history": "^4.10.1",
|
||||
"local-storage-fallback": "^4.1.1",
|
||||
@@ -28,6 +29,7 @@
|
||||
"moment": "^2.24.0",
|
||||
"npm": "^6.14.4",
|
||||
"react": "^16.13.1",
|
||||
"react-chartjs-2": "^2.9.0",
|
||||
"react-codemirror2": "^7.1.0",
|
||||
"react-dom": "^16.12.0",
|
||||
"react-moment": "^0.9.7",
|
||||
|
||||
@@ -62,6 +62,7 @@ import { Button, LinearProgress } from "@material-ui/core";
|
||||
import WebhookPanel from "./Configurations/ConfigurationPanels/WebhookPanel";
|
||||
import Trace from "./Trace/Trace";
|
||||
import Logs from "./Logs/Logs";
|
||||
import Heal from "./Heal/Heal";
|
||||
import Watch from "./Watch/Watch";
|
||||
import ListClusters from "./Clusters/ListClusters/ListClusters";
|
||||
import { ISessionResponse } from "./types";
|
||||
@@ -271,6 +272,10 @@ const Console = ({
|
||||
component: Logs,
|
||||
path: "/logs",
|
||||
},
|
||||
{
|
||||
component: Heal,
|
||||
path: "/heal",
|
||||
},
|
||||
{
|
||||
component: ListNotificationEndpoints,
|
||||
path: "/notification-endpoints",
|
||||
|
||||
327
portal-ui/src/screens/Console/Heal/Heal.tsx
Normal file
327
portal-ui/src/screens/Console/Heal/Heal.tsx
Normal file
@@ -0,0 +1,327 @@
|
||||
import { HorizontalBar } from "react-chartjs-2";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import {
|
||||
Button,
|
||||
Grid,
|
||||
Typography,
|
||||
TextField,
|
||||
Checkbox,
|
||||
} from "@material-ui/core";
|
||||
import { IMessageEvent, w3cwebsocket as W3CWebSocket } from "websocket";
|
||||
import { createStyles, Theme, withStyles } from "@material-ui/core/styles";
|
||||
import { wsProtocol } from "../../../utils/wsUtils";
|
||||
import api from "../../../common/api";
|
||||
import { FormControl, MenuItem, Select } from "@material-ui/core";
|
||||
import { BucketList, Bucket } from "../Watch/types";
|
||||
import { HealStatus, colorH } from "./types";
|
||||
import { niceBytes } from "../../../common/utils";
|
||||
|
||||
const styles = (theme: Theme) =>
|
||||
createStyles({
|
||||
watchList: {
|
||||
background: "white",
|
||||
maxHeight: "400",
|
||||
overflow: "auto",
|
||||
"& ul": {
|
||||
margin: "4",
|
||||
padding: "0",
|
||||
},
|
||||
"& ul li": {
|
||||
listStyle: "none",
|
||||
margin: "0",
|
||||
padding: "0",
|
||||
borderBottom: "1px solid #dedede",
|
||||
},
|
||||
},
|
||||
actionsTray: {
|
||||
textAlign: "right",
|
||||
"& button": {
|
||||
marginLeft: 10,
|
||||
},
|
||||
},
|
||||
inputField: {
|
||||
background: "#FFFFFF",
|
||||
padding: 12,
|
||||
borderRadius: 5,
|
||||
marginLeft: 10,
|
||||
boxShadow: "0px 3px 6px #00000012",
|
||||
},
|
||||
fieldContainer: {
|
||||
background: "#FFFFFF",
|
||||
padding: 0,
|
||||
borderRadius: 5,
|
||||
marginLeft: 10,
|
||||
textAlign: "left",
|
||||
minWidth: "206",
|
||||
boxShadow: "0px 3px 6px #00000012",
|
||||
},
|
||||
lastElementWPadding: {
|
||||
paddingRight: "78",
|
||||
},
|
||||
});
|
||||
|
||||
interface IHeal {
|
||||
classes: any;
|
||||
}
|
||||
|
||||
const Heal = ({ classes }: IHeal) => {
|
||||
const [start, setStart] = useState(false);
|
||||
const [bucketName, setBucketName] = useState("Select Bucket");
|
||||
const [bucketList, setBucketList] = useState<Bucket[]>([]);
|
||||
const [prefix, setPrefix] = useState("");
|
||||
const [recursive, setRecursive] = useState(false);
|
||||
const [forceStart, setForceStart] = useState(false);
|
||||
const [forceStop, setForceStop] = useState(false);
|
||||
// healStatus states
|
||||
const [hStatus, setHStatus] = useState({
|
||||
beforeHeal: [0, 0, 0, 0],
|
||||
afterHeal: [0, 0, 0, 0],
|
||||
objectsHealed: 0,
|
||||
objectsScanned: 0,
|
||||
healDuration: 0,
|
||||
sizeScanned: "",
|
||||
});
|
||||
|
||||
const fetchBucketList = () => {
|
||||
api
|
||||
.invoke("GET", `/api/v1/buckets`)
|
||||
.then((res: BucketList) => {
|
||||
let buckets: Bucket[] = [];
|
||||
if (res.buckets !== null) {
|
||||
buckets = res.buckets;
|
||||
}
|
||||
setBucketList(buckets);
|
||||
})
|
||||
.catch((err: any) => {
|
||||
console.log(err);
|
||||
});
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
fetchBucketList();
|
||||
}, []);
|
||||
|
||||
// forceStart and forceStop need to be mutually exclusive
|
||||
useEffect(() => {
|
||||
if (forceStart === true) {
|
||||
setForceStop(false);
|
||||
}
|
||||
}, [forceStart]);
|
||||
|
||||
useEffect(() => {
|
||||
if (forceStop === true) {
|
||||
setForceStart(false);
|
||||
}
|
||||
}, [forceStop]);
|
||||
|
||||
const colorHealthArr = (color: colorH) => {
|
||||
return [color.Green, color.Yellow, color.Red, color.Grey];
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
// begin watch if bucketName in bucketList and start pressed
|
||||
if (start) {
|
||||
// values stored here to update chart
|
||||
const cB: colorH = { Green: 0, Yellow: 0, Red: 0, Grey: 0 };
|
||||
const cA: colorH = { Green: 0, Yellow: 0, Red: 0, Grey: 0 };
|
||||
|
||||
const url = new URL(window.location.toString());
|
||||
const isDev = process.env.NODE_ENV === "development";
|
||||
const port = isDev ? "9090" : url.port;
|
||||
|
||||
const wsProt = wsProtocol(url.protocol);
|
||||
const c = new W3CWebSocket(
|
||||
`${wsProt}://${url.hostname}:${port}/ws/heal/${bucketName}?prefix=${prefix}&recursive=${recursive}&force-start=${forceStart}&force-stop=${forceStop}`
|
||||
);
|
||||
|
||||
if (c !== null) {
|
||||
c.onopen = () => {
|
||||
console.log("WebSocket Client Connected");
|
||||
c.send("ok");
|
||||
};
|
||||
c.onmessage = (message: IMessageEvent) => {
|
||||
let m: HealStatus = JSON.parse(message.data.toString());
|
||||
// Store percentage per health color
|
||||
for (const [key, value] of Object.entries(m.healthAfterCols)) {
|
||||
cA[key] = (value * 100) / m.itemsScanned;
|
||||
}
|
||||
for (const [key, value] of Object.entries(m.healthBeforeCols)) {
|
||||
cB[key] = (value * 100) / m.itemsScanned;
|
||||
}
|
||||
setHStatus({
|
||||
beforeHeal: colorHealthArr(cB),
|
||||
afterHeal: colorHealthArr(cA),
|
||||
objectsHealed: m.objectsHealed,
|
||||
objectsScanned: m.objectsScanned,
|
||||
healDuration: m.healDuration,
|
||||
sizeScanned: niceBytes(m.bytesScanned.toString()),
|
||||
});
|
||||
};
|
||||
c.onclose = () => {
|
||||
setStart(false);
|
||||
console.log("connection closed by server");
|
||||
};
|
||||
return () => {
|
||||
// close websocket on useEffect cleanup
|
||||
c.close(1000);
|
||||
console.log("closing websockets");
|
||||
};
|
||||
}
|
||||
}
|
||||
}, [start]);
|
||||
|
||||
let data = {
|
||||
labels: ["Green", "Yellow", "Red", "Grey"],
|
||||
datasets: [
|
||||
{
|
||||
label: "After Healing",
|
||||
data: hStatus.afterHeal,
|
||||
backgroundColor: "rgba(0, 0, 255, 0.2)",
|
||||
borderColor: "rgba(54, 162, 235, 1)",
|
||||
borderWidth: 1,
|
||||
},
|
||||
{
|
||||
label: "Before Healing",
|
||||
data: hStatus.beforeHeal,
|
||||
backgroundColor: "rgba(153, 102, 255, 0.2)",
|
||||
borderColor: "rgba(153, 102, 255, 1)",
|
||||
borderWidth: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
const bucketNames = bucketList.map((bucketName) => ({
|
||||
label: bucketName.name,
|
||||
value: bucketName.name,
|
||||
}));
|
||||
return (
|
||||
<React.Fragment>
|
||||
<Grid container>
|
||||
<Grid item xs={12}>
|
||||
<Typography variant="h6">Heal</Typography>
|
||||
</Grid>
|
||||
<Grid item xs={12}>
|
||||
<br />
|
||||
</Grid>
|
||||
<Grid item xs={12} className={classes.actionsTray}>
|
||||
<FormControl variant="outlined">
|
||||
<Select
|
||||
id="bucket-name"
|
||||
name="bucket-name"
|
||||
value={bucketName}
|
||||
onChange={(e) => {
|
||||
setBucketName(e.target.value as string);
|
||||
}}
|
||||
className={classes.fieldContainer}
|
||||
disabled={false}
|
||||
>
|
||||
<MenuItem value="" key={`select-bucket-name-default`}>
|
||||
Select Bucket
|
||||
</MenuItem>
|
||||
{bucketNames.map((option) => (
|
||||
<MenuItem
|
||||
value={option.value}
|
||||
key={`select-bucket-name-${option.label}`}
|
||||
>
|
||||
{option.label}
|
||||
</MenuItem>
|
||||
))}
|
||||
</Select>
|
||||
</FormControl>
|
||||
<TextField
|
||||
placeholder="Prefix"
|
||||
className={classes.inputField}
|
||||
id="prefix-resource"
|
||||
label=""
|
||||
disabled={false}
|
||||
InputProps={{
|
||||
disableUnderline: true,
|
||||
}}
|
||||
onChange={(e) => {
|
||||
setPrefix(e.target.value);
|
||||
}}
|
||||
/>
|
||||
<Button
|
||||
type="submit"
|
||||
variant="contained"
|
||||
color="primary"
|
||||
disabled={start}
|
||||
onClick={() => setStart(true)}
|
||||
>
|
||||
Start
|
||||
</Button>
|
||||
<Grid item xs={12}>
|
||||
<span>{"Recursive"}</span>
|
||||
<Checkbox
|
||||
name="recursive"
|
||||
id="recursive"
|
||||
value="recursive"
|
||||
color="primary"
|
||||
inputProps={{ "aria-label": "secondary checkbox" }}
|
||||
checked={recursive}
|
||||
onChange={(e) => {
|
||||
setRecursive(e.target.checked);
|
||||
}}
|
||||
disabled={false}
|
||||
/>
|
||||
<span>{"Force Start"}</span>
|
||||
<Checkbox
|
||||
name="recursive"
|
||||
id="recursive"
|
||||
value="recursive"
|
||||
color="primary"
|
||||
inputProps={{ "aria-label": "secondary checkbox" }}
|
||||
checked={forceStart}
|
||||
onChange={(e) => {
|
||||
setForceStart(e.target.checked);
|
||||
}}
|
||||
disabled={false}
|
||||
/>
|
||||
<span>{"Force Stop"}</span>
|
||||
<Checkbox
|
||||
name="recursive"
|
||||
id="recursive"
|
||||
value="recursive"
|
||||
color="primary"
|
||||
inputProps={{ "aria-label": "secondary checkbox" }}
|
||||
checked={forceStop}
|
||||
onChange={(e) => {
|
||||
setForceStop(e.target.checked);
|
||||
}}
|
||||
disabled={false}
|
||||
/>
|
||||
<span className={classes.lastElementWPadding}>{""}</span>
|
||||
</Grid>
|
||||
</Grid>
|
||||
<Grid item xs={12}>
|
||||
<br />
|
||||
</Grid>
|
||||
<HorizontalBar
|
||||
data={data}
|
||||
width={100}
|
||||
height={30}
|
||||
options={{
|
||||
title: {
|
||||
display: true,
|
||||
text: "Item's Health Status [%]",
|
||||
fontSize: 20,
|
||||
},
|
||||
legend: {
|
||||
display: true,
|
||||
position: "right",
|
||||
},
|
||||
}}
|
||||
/>
|
||||
<Grid item xs={12}>
|
||||
<br />
|
||||
Size scanned: {hStatus.sizeScanned}
|
||||
<br />
|
||||
Objects healed: {hStatus.objectsHealed} / {hStatus.objectsScanned}
|
||||
<br />
|
||||
Healing time: {hStatus.healDuration}s
|
||||
</Grid>
|
||||
</Grid>
|
||||
</React.Fragment>
|
||||
);
|
||||
};
|
||||
|
||||
export default withStyles(styles)(Heal);
|
||||
64
portal-ui/src/screens/Console/Heal/types.ts
Normal file
64
portal-ui/src/screens/Console/Heal/types.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
// This file is part of MinIO Console Server
|
||||
// Copyright (c) 2020 MinIO, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
export interface HealDriveInfo {
|
||||
uuid: string;
|
||||
endpoint: string;
|
||||
state: string;
|
||||
}
|
||||
|
||||
export interface MomentHealth {
|
||||
color: string;
|
||||
offline: number;
|
||||
online: number;
|
||||
missing: number;
|
||||
corrupted: number;
|
||||
drives: HealDriveInfo[];
|
||||
}
|
||||
|
||||
export interface HealItemStatus {
|
||||
status: string;
|
||||
error: string;
|
||||
type: string;
|
||||
name: string;
|
||||
before: MomentHealth;
|
||||
after: MomentHealth;
|
||||
size: number;
|
||||
}
|
||||
|
||||
export interface HealStatus {
|
||||
healDuration: number;
|
||||
bytesScanned: number;
|
||||
objectsScanned: number;
|
||||
itemsScanned: number;
|
||||
// Counters for healed objects and all kinds of healed items
|
||||
objectsHealed: number;
|
||||
itemsHealed: number;
|
||||
|
||||
itemsHealthStatus: HealItemStatus[];
|
||||
// Map of health color code to number of objects with that
|
||||
// health color code.
|
||||
healthBeforeCols: Map<string, number>;
|
||||
healthAfterCols: Map<string, number>;
|
||||
}
|
||||
|
||||
// colorH used to save health's percentage per color
|
||||
export interface colorH {
|
||||
[Green: string]: number;
|
||||
Yellow: number;
|
||||
Red: number;
|
||||
Grey: number;
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import ListItem from "@material-ui/core/ListItem";
|
||||
import ListItemIcon from "@material-ui/core/ListItemIcon";
|
||||
import RoomServiceIcon from "@material-ui/icons/RoomService";
|
||||
import WebAssetIcon from "@material-ui/icons/WebAsset";
|
||||
import HealingIcon from "@material-ui/icons/Healing";
|
||||
import CenterFocusWeakIcon from "@material-ui/icons/CenterFocusWeak";
|
||||
import StorageIcon from "@material-ui/icons/Storage";
|
||||
import ListItemText from "@material-ui/core/ListItemText";
|
||||
@@ -202,6 +203,14 @@ class Menu extends React.Component<MenuProps> {
|
||||
name: "Console Logs",
|
||||
icon: <WebAssetIcon />,
|
||||
},
|
||||
{
|
||||
group: "Admin",
|
||||
type: "item",
|
||||
component: NavLink,
|
||||
to: "/heal",
|
||||
name: "Heal",
|
||||
icon: <HealingIcon />,
|
||||
},
|
||||
{
|
||||
type: "title",
|
||||
name: "Configuration",
|
||||
|
||||
@@ -3013,6 +3013,29 @@ chardet@^0.7.0:
|
||||
resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
|
||||
integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==
|
||||
|
||||
chart.js@^2.9.3:
|
||||
version "2.9.3"
|
||||
resolved "https://registry.yarnpkg.com/chart.js/-/chart.js-2.9.3.tgz#ae3884114dafd381bc600f5b35a189138aac1ef7"
|
||||
integrity sha512-+2jlOobSk52c1VU6fzkh3UwqHMdSlgH1xFv9FKMqHiNCpXsGPQa/+81AFa+i3jZ253Mq9aAycPwDjnn1XbRNNw==
|
||||
dependencies:
|
||||
chartjs-color "^2.1.0"
|
||||
moment "^2.10.2"
|
||||
|
||||
chartjs-color-string@^0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz#1df096621c0e70720a64f4135ea171d051402f71"
|
||||
integrity sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==
|
||||
dependencies:
|
||||
color-name "^1.0.0"
|
||||
|
||||
chartjs-color@^2.1.0:
|
||||
version "2.4.1"
|
||||
resolved "https://registry.yarnpkg.com/chartjs-color/-/chartjs-color-2.4.1.tgz#6118bba202fe1ea79dd7f7c0f9da93467296c3b0"
|
||||
integrity sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==
|
||||
dependencies:
|
||||
chartjs-color-string "^0.6.0"
|
||||
color-convert "^1.9.3"
|
||||
|
||||
chokidar@^2.1.8:
|
||||
version "2.1.8"
|
||||
resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917"
|
||||
@@ -3243,7 +3266,7 @@ collection-visit@^1.0.0:
|
||||
map-visit "^1.0.0"
|
||||
object-visit "^1.0.0"
|
||||
|
||||
color-convert@^1.9.0, color-convert@^1.9.1:
|
||||
color-convert@^1.9.0, color-convert@^1.9.1, color-convert@^1.9.3:
|
||||
version "1.9.3"
|
||||
resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
|
||||
integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
|
||||
@@ -7632,7 +7655,7 @@ lodash.without@~4.4.0:
|
||||
resolved "https://registry.yarnpkg.com/lodash.without/-/lodash.without-4.4.0.tgz#3cd4574a00b67bae373a94b748772640507b7aac"
|
||||
integrity sha1-PNRXSgC2e643OpS3SHcmQFB7eqw=
|
||||
|
||||
"lodash@>=3.5 <5", lodash@^4.17.11, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.5, lodash@~4.17.4:
|
||||
"lodash@>=3.5 <5", lodash@^4.17.11, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.4, lodash@^4.17.5, lodash@~4.17.4:
|
||||
version "4.17.15"
|
||||
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
|
||||
integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
|
||||
@@ -8028,6 +8051,11 @@ mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.4, mkdirp@~0.5.0, mkdir
|
||||
dependencies:
|
||||
minimist "^1.2.5"
|
||||
|
||||
moment@^2.10.2:
|
||||
version "2.26.0"
|
||||
resolved "https://registry.yarnpkg.com/moment/-/moment-2.26.0.tgz#5e1f82c6bafca6e83e808b30c8705eed0dcbd39a"
|
||||
integrity sha512-oIixUO+OamkUkwjhAVE18rAMfRJNsNe/Stid/gwHSOfHrOtw9EhAY2AHvdKZ/k/MggcYELFCJz/Sn2pL8b8JMw==
|
||||
|
||||
moment@^2.24.0:
|
||||
version "2.24.0"
|
||||
resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
|
||||
@@ -9959,7 +9987,7 @@ promzard@^0.3.0:
|
||||
dependencies:
|
||||
read "1"
|
||||
|
||||
prop-types@^15.6.0, prop-types@^15.6.2, prop-types@^15.7.2:
|
||||
prop-types@^15.5.8, prop-types@^15.6.0, prop-types@^15.6.2, prop-types@^15.7.2:
|
||||
version "15.7.2"
|
||||
resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.7.2.tgz#52c41e75b8c87e72b9d9360e0206b99dcbffa6c5"
|
||||
integrity sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==
|
||||
@@ -10176,6 +10204,14 @@ react-app-polyfill@^1.0.6:
|
||||
regenerator-runtime "^0.13.3"
|
||||
whatwg-fetch "^3.0.0"
|
||||
|
||||
react-chartjs-2@^2.9.0:
|
||||
version "2.9.0"
|
||||
resolved "https://registry.yarnpkg.com/react-chartjs-2/-/react-chartjs-2-2.9.0.tgz#d054dbdd763fbe9a76296a4ae0752ea549b76d9e"
|
||||
integrity sha512-IYwqUUnQRAJ9SNA978vxulHJTcUFTJk2LDVfbAyk0TnJFZZG7+6U/2flsE4MCw6WCbBjTTypy8T82Ch7XrPtRw==
|
||||
dependencies:
|
||||
lodash "^4.17.4"
|
||||
prop-types "^15.5.8"
|
||||
|
||||
react-codemirror2@^7.1.0:
|
||||
version "7.1.0"
|
||||
resolved "https://registry.yarnpkg.com/react-codemirror2/-/react-codemirror2-7.1.0.tgz#b874a275ad4f6f2ee5adb23b550c0f4b8b82776d"
|
||||
|
||||
378
restapi/admin_heal.go
Normal file
378
restapi/admin_heal.go
Normal file
@@ -0,0 +1,378 @@
|
||||
// This file is part of MinIO Console Server
|
||||
// Copyright (c) 2020 MinIO, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package restapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
// An alias of string to represent the health color code of an object
|
||||
type col string
|
||||
|
||||
const (
|
||||
colGrey col = "Grey"
|
||||
colRed col = "Red"
|
||||
colYellow col = "Yellow"
|
||||
colGreen col = "Green"
|
||||
)
|
||||
|
||||
var (
|
||||
hColOrder = []col{colRed, colYellow, colGreen}
|
||||
hColTable = map[int][]int{
|
||||
1: {0, -1, 1},
|
||||
2: {0, 1, 2},
|
||||
3: {1, 2, 3},
|
||||
4: {1, 2, 4},
|
||||
5: {1, 3, 5},
|
||||
6: {2, 4, 6},
|
||||
7: {2, 4, 7},
|
||||
8: {2, 5, 8},
|
||||
}
|
||||
)
|
||||
|
||||
type healItemStatus struct {
|
||||
Status string `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
Before struct {
|
||||
Color string `json:"color"`
|
||||
Offline int `json:"offline"`
|
||||
Online int `json:"online"`
|
||||
Missing int `json:"missing"`
|
||||
Corrupted int `json:"corrupted"`
|
||||
Drives []madmin.HealDriveInfo `json:"drives"`
|
||||
} `json:"before"`
|
||||
After struct {
|
||||
Color string `json:"color"`
|
||||
Offline int `json:"offline"`
|
||||
Online int `json:"online"`
|
||||
Missing int `json:"missing"`
|
||||
Corrupted int `json:"corrupted"`
|
||||
Drives []madmin.HealDriveInfo `json:"drives"`
|
||||
} `json:"after"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type healStatus struct {
|
||||
// Total time since heal start in seconds
|
||||
HealDuration float64 `json:"healDuration"`
|
||||
|
||||
// Accumulated statistics of heal result records
|
||||
BytesScanned int64 `json:"bytesScanned"`
|
||||
|
||||
// Counter for objects, and another counter for all kinds of
|
||||
// items
|
||||
ObjectsScanned int64 `json:"objectsScanned"`
|
||||
ItemsScanned int64 `json:"itemsScanned"`
|
||||
|
||||
// Counters for healed objects and all kinds of healed items
|
||||
ObjectsHealed int64 `json:"objectsHealed"`
|
||||
ItemsHealed int64 `json:"itemsHealed"`
|
||||
|
||||
ItemsHealthStatus []healItemStatus `json:"itemsHealthStatus"`
|
||||
// Map of health color code to number of objects with that
|
||||
// health color code.
|
||||
HealthBeforeCols map[col]int64 `json:"healthBeforeCols"`
|
||||
HealthAfterCols map[col]int64 `json:"healthAfterCols"`
|
||||
}
|
||||
|
||||
type healOptions struct {
|
||||
BucketName string
|
||||
Prefix string
|
||||
ForceStart bool
|
||||
ForceStop bool
|
||||
madmin.HealOpts
|
||||
}
|
||||
|
||||
// startHeal starts healing of the servers based on heal options
|
||||
func startHeal(ctx context.Context, conn WSConn, client MinioAdmin, hOpts *healOptions) error {
|
||||
// Initialize heal
|
||||
healStart, _, err := client.heal(ctx, hOpts.BucketName, hOpts.Prefix, hOpts.HealOpts, "", hOpts.ForceStart, hOpts.ForceStop)
|
||||
if err != nil {
|
||||
log.Println("error initializing healing:", err)
|
||||
return err
|
||||
}
|
||||
if hOpts.ForceStop {
|
||||
log.Println("heal stopped successfully")
|
||||
return nil
|
||||
}
|
||||
clientToken := healStart.ClientToken
|
||||
hs := healStatus{
|
||||
HealthBeforeCols: make(map[col]int64),
|
||||
HealthAfterCols: make(map[col]int64),
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
_, res, err := client.heal(ctx, hOpts.BucketName, hOpts.Prefix, hOpts.HealOpts, clientToken, hOpts.ForceStart, hOpts.ForceStop)
|
||||
if err != nil {
|
||||
log.Println("error on heal:", err)
|
||||
return err
|
||||
}
|
||||
|
||||
hs.writeStatus(&res, conn)
|
||||
|
||||
if res.Summary == "finished" {
|
||||
log.Println("heal finished")
|
||||
return nil
|
||||
}
|
||||
|
||||
if res.Summary == "stopped" {
|
||||
log.Println("heal stopped")
|
||||
return fmt.Errorf("heal had an error - %s", res.FailureDetail)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healStatus) writeStatus(s *madmin.HealTaskStatus, conn WSConn) error {
|
||||
// Update state
|
||||
h.updateDuration(s)
|
||||
for _, item := range s.Items {
|
||||
err := h.updateStats(item)
|
||||
if err != nil {
|
||||
fmt.Println("error on updateStats:", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize message to be sent
|
||||
infoBytes, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
fmt.Println("error on json.Marshal:", err)
|
||||
return err
|
||||
}
|
||||
// Send Message through websocket connection
|
||||
err = conn.writeMessage(websocket.TextMessage, infoBytes)
|
||||
if err != nil {
|
||||
log.Println("error writeMessage:", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *healStatus) updateDuration(s *madmin.HealTaskStatus) {
|
||||
h.HealDuration = time.Now().UTC().Sub(s.StartTime).Round(time.Second).Seconds()
|
||||
}
|
||||
|
||||
func (h *healStatus) updateStats(i madmin.HealResultItem) error {
|
||||
// update general status
|
||||
if i.Type == madmin.HealItemObject {
|
||||
// Objects whose size could not be found have -1 size
|
||||
// returned.
|
||||
if i.ObjectSize >= 0 {
|
||||
h.BytesScanned += i.ObjectSize
|
||||
}
|
||||
h.ObjectsScanned++
|
||||
}
|
||||
h.ItemsScanned++
|
||||
|
||||
beforeUp, afterUp := i.GetOnlineCounts()
|
||||
if afterUp > beforeUp {
|
||||
if i.Type == madmin.HealItemObject {
|
||||
h.ObjectsHealed++
|
||||
}
|
||||
h.ItemsHealed++
|
||||
}
|
||||
// update per item status
|
||||
itemStatus := healItemStatus{}
|
||||
// get color health status
|
||||
var beforeColor, afterColor col
|
||||
var err error
|
||||
switch i.Type {
|
||||
case madmin.HealItemMetadata, madmin.HealItemBucket:
|
||||
beforeColor, afterColor, err = getReplicatedFileHCCChange(i)
|
||||
default:
|
||||
if i.Type == madmin.HealItemObject {
|
||||
itemStatus.Size = i.ObjectSize
|
||||
}
|
||||
beforeColor, afterColor, err = getObjectHCCChange(i)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
itemStatus.Status = "success"
|
||||
itemStatus.Before.Color = strings.ToLower(string(beforeColor))
|
||||
itemStatus.After.Color = strings.ToLower(string(afterColor))
|
||||
itemStatus.Type, itemStatus.Name = getHRITypeAndName(i)
|
||||
itemStatus.Before.Online, itemStatus.After.Online = beforeUp, afterUp
|
||||
itemStatus.Before.Missing, itemStatus.After.Missing = i.GetMissingCounts()
|
||||
itemStatus.Before.Corrupted, itemStatus.After.Corrupted = i.GetCorruptedCounts()
|
||||
itemStatus.Before.Offline, itemStatus.After.Offline = i.GetOfflineCounts()
|
||||
itemStatus.Before.Drives = i.Before.Drives
|
||||
itemStatus.After.Drives = i.After.Drives
|
||||
h.ItemsHealthStatus = append(h.ItemsHealthStatus, itemStatus)
|
||||
h.HealthBeforeCols[beforeColor]++
|
||||
h.HealthAfterCols[afterColor]++
|
||||
return nil
|
||||
}
|
||||
|
||||
// getObjectHCCChange - returns before and after color change for
|
||||
// objects
|
||||
func getObjectHCCChange(h madmin.HealResultItem) (b, a col, err error) {
|
||||
parityShards := h.ParityBlocks
|
||||
dataShards := h.DataBlocks
|
||||
|
||||
onlineBefore, onlineAfter := h.GetOnlineCounts()
|
||||
surplusShardsBeforeHeal := onlineBefore - dataShards
|
||||
surplusShardsAfterHeal := onlineAfter - dataShards
|
||||
b, err = getHColCode(surplusShardsBeforeHeal, parityShards)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
a, err = getHColCode(surplusShardsAfterHeal, parityShards)
|
||||
return
|
||||
|
||||
}
|
||||
|
||||
// getReplicatedFileHCCChange - fetches health color code for metadata
|
||||
// files that are replicated.
|
||||
func getReplicatedFileHCCChange(h madmin.HealResultItem) (b, a col, err error) {
|
||||
getColCode := func(numAvail int) (c col, err error) {
|
||||
// calculate color code for replicated object similar
|
||||
// to erasure coded objects
|
||||
quorum := h.DiskCount/h.SetCount/2 + 1
|
||||
surplus := numAvail/h.SetCount - quorum
|
||||
parity := h.DiskCount/h.SetCount - quorum
|
||||
c, err = getHColCode(surplus, parity)
|
||||
return
|
||||
}
|
||||
|
||||
onlineBefore, onlineAfter := h.GetOnlineCounts()
|
||||
b, err = getColCode(onlineBefore)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
a, err = getColCode(onlineAfter)
|
||||
return
|
||||
}
|
||||
|
||||
func getHColCode(surplusShards, parityShards int) (c col, err error) {
|
||||
if parityShards < 1 || parityShards > 8 || surplusShards > parityShards {
|
||||
return c, fmt.Errorf("invalid parity shard count/surplus shard count given")
|
||||
}
|
||||
if surplusShards < 0 {
|
||||
return colGrey, err
|
||||
}
|
||||
colRow := hColTable[parityShards]
|
||||
for index, val := range colRow {
|
||||
if val != -1 && surplusShards <= val {
|
||||
return hColOrder[index], err
|
||||
}
|
||||
}
|
||||
return c, fmt.Errorf("cannot get a heal color code")
|
||||
}
|
||||
|
||||
func getHRITypeAndName(i madmin.HealResultItem) (typ, name string) {
|
||||
name = fmt.Sprintf("%s/%s", i.Bucket, i.Object)
|
||||
switch i.Type {
|
||||
case madmin.HealItemMetadata:
|
||||
typ = "system"
|
||||
name = i.Detail
|
||||
case madmin.HealItemBucketMetadata:
|
||||
typ = "system"
|
||||
name = "bucket-metadata:" + name
|
||||
case madmin.HealItemBucket:
|
||||
typ = "bucket"
|
||||
case madmin.HealItemObject:
|
||||
typ = "object"
|
||||
default:
|
||||
typ = fmt.Sprintf("!! Unknown heal result record %#v !!", i)
|
||||
name = typ
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// getHealOptionsFromReq return options from request for healing process
|
||||
// path come as : `/heal/bucket1` and query params come on request form
|
||||
func getHealOptionsFromReq(req *http.Request) (*healOptions, error) {
|
||||
hOptions := healOptions{}
|
||||
re := regexp.MustCompile(`(/heal/)(.*?)(\?.*?$|$)`)
|
||||
matches := re.FindAllSubmatch([]byte(req.URL.Path), -1)
|
||||
// len matches is always 3
|
||||
// matches comes as e.g.
|
||||
// [["...", "/heal/" "bucket1"]]
|
||||
// [["/heal/" "/heal/" ""]]
|
||||
|
||||
// bucket name is on the second group, third position
|
||||
hOptions.BucketName = strings.TrimSpace(string(matches[0][2]))
|
||||
hOptions.Prefix = req.FormValue("prefix")
|
||||
hOptions.HealOpts.ScanMode = transformScanStr(req.FormValue("scan"))
|
||||
|
||||
if req.FormValue("force-start") != "" {
|
||||
boolVal, err := strconv.ParseBool(req.FormValue("force-start"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hOptions.ForceStart = boolVal
|
||||
}
|
||||
if req.FormValue("force-stop") != "" {
|
||||
boolVal, err := strconv.ParseBool(req.FormValue("force-stop"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hOptions.ForceStop = boolVal
|
||||
}
|
||||
// heal recursively
|
||||
if req.FormValue("recursive") != "" {
|
||||
boolVal, err := strconv.ParseBool(req.FormValue("recursive"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hOptions.HealOpts.Recursive = boolVal
|
||||
}
|
||||
// remove dangling objects in heal sequence
|
||||
if req.FormValue("remove") != "" {
|
||||
boolVal, err := strconv.ParseBool(req.FormValue("remove"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hOptions.HealOpts.Remove = boolVal
|
||||
}
|
||||
// only inspect data
|
||||
if req.FormValue("dry-run") != "" {
|
||||
boolVal, err := strconv.ParseBool(req.FormValue("dry-run"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hOptions.HealOpts.DryRun = boolVal
|
||||
}
|
||||
return &hOptions, nil
|
||||
}
|
||||
|
||||
func transformScanStr(scanStr string) madmin.HealScanMode {
|
||||
switch scanStr {
|
||||
case "deep":
|
||||
return madmin.HealDeepScan
|
||||
}
|
||||
return madmin.HealNormalScan
|
||||
}
|
||||
278
restapi/admin_heal_test.go
Normal file
278
restapi/admin_heal_test.go
Normal file
@@ -0,0 +1,278 @@
|
||||
// This file is part of MinIO Console Server
|
||||
// Copyright (c) 2020 MinIO, Inc.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package restapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// assigning mock at runtime instead of compile time
|
||||
var minioHealMock func(ctx context.Context, bucket, prefix string, healOpts madmin.HealOpts, clientToken string,
|
||||
forceStart, forceStop bool) (healStart madmin.HealStartSuccess, healTaskStatus madmin.HealTaskStatus, err error)
|
||||
|
||||
func (ac adminClientMock) heal(ctx context.Context, bucket, prefix string, healOpts madmin.HealOpts, clientToken string,
|
||||
forceStart, forceStop bool) (healStart madmin.HealStartSuccess, healTaskStatus madmin.HealTaskStatus, err error) {
|
||||
return minioHealMock(ctx, bucket, prefix, healOpts, clientToken, forceStart, forceStop)
|
||||
}
|
||||
|
||||
func TestHeal(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
client := adminClientMock{}
|
||||
mockWSConn := mockConn{}
|
||||
ctx := context.Background()
|
||||
|
||||
function := "startHeal()"
|
||||
mockResultItem1 := madmin.HealResultItem{
|
||||
Type: madmin.HealItemObject,
|
||||
SetCount: 1,
|
||||
DiskCount: 4,
|
||||
ParityBlocks: 2,
|
||||
DataBlocks: 2,
|
||||
Before: struct {
|
||||
Drives []madmin.HealDriveInfo `json:"drives"`
|
||||
}{
|
||||
Drives: []madmin.HealDriveInfo{
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateMissing,
|
||||
},
|
||||
},
|
||||
},
|
||||
After: struct {
|
||||
Drives []madmin.HealDriveInfo `json:"drives"`
|
||||
}{
|
||||
Drives: []madmin.HealDriveInfo{
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
mockResultItem2 := madmin.HealResultItem{
|
||||
Type: madmin.HealItemBucket,
|
||||
SetCount: 1,
|
||||
DiskCount: 4,
|
||||
ParityBlocks: 2,
|
||||
DataBlocks: 2,
|
||||
Before: struct {
|
||||
Drives []madmin.HealDriveInfo `json:"drives"`
|
||||
}{
|
||||
Drives: []madmin.HealDriveInfo{
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateMissing,
|
||||
},
|
||||
},
|
||||
},
|
||||
After: struct {
|
||||
Drives []madmin.HealDriveInfo `json:"drives"`
|
||||
}{
|
||||
Drives: []madmin.HealDriveInfo{
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
{
|
||||
State: madmin.DriveStateOk,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
mockHealTaskStatus := madmin.HealTaskStatus{
|
||||
StartTime: time.Now().UTC().Truncate(time.Second * 2), // mock 2 sec duration
|
||||
Items: []madmin.HealResultItem{
|
||||
mockResultItem1,
|
||||
mockResultItem2,
|
||||
},
|
||||
Summary: "finished",
|
||||
}
|
||||
|
||||
testStreamSize := 1
|
||||
testReceiver := make(chan healStatus, testStreamSize)
|
||||
isClosed := false // testReceiver is closed?
|
||||
|
||||
testOptions := &healOptions{
|
||||
BucketName: "testbucket",
|
||||
Prefix: "",
|
||||
ForceStart: false,
|
||||
ForceStop: false,
|
||||
}
|
||||
// Test-1: startHeal send simple stream of data, no errors
|
||||
minioHealMock = func(ctx context.Context, bucket, prefix string, healOpts madmin.HealOpts, clientToken string,
|
||||
forceStart, forceStop bool) (healStart madmin.HealStartSuccess, healTaskStatus madmin.HealTaskStatus, err error) {
|
||||
|
||||
return healStart, mockHealTaskStatus, nil
|
||||
}
|
||||
writesCount := 1
|
||||
// mock connection WriteMessage() no error
|
||||
connWriteMessageMock = func(messageType int, data []byte) error {
|
||||
// emulate that receiver gets the message written
|
||||
var t healStatus
|
||||
_ = json.Unmarshal(data, &t)
|
||||
testReceiver <- t
|
||||
if writesCount == testStreamSize {
|
||||
// for testing we need to close the receiver channel
|
||||
if !isClosed {
|
||||
close(testReceiver)
|
||||
isClosed = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
writesCount++
|
||||
return nil
|
||||
}
|
||||
if err := startHeal(ctx, mockWSConn, client, testOptions); err != nil {
|
||||
t.Errorf("Failed on %s:, error occurred: %s", function, err.Error())
|
||||
}
|
||||
// check that the TestReceiver got the same number of data from Console.
|
||||
for i := range testReceiver {
|
||||
assert.Equal(int64(1), i.ObjectsScanned)
|
||||
assert.Equal(int64(1), i.ObjectsHealed)
|
||||
assert.Equal(int64(2), i.ItemsScanned)
|
||||
assert.Equal(int64(2), i.ItemsHealed)
|
||||
assert.Equal(int64(0), i.HealthBeforeCols[colGreen])
|
||||
assert.Equal(int64(1), i.HealthBeforeCols[colYellow])
|
||||
assert.Equal(int64(1), i.HealthBeforeCols[colRed])
|
||||
assert.Equal(int64(0), i.HealthBeforeCols[colGrey])
|
||||
assert.Equal(int64(2), i.HealthAfterCols[colGreen])
|
||||
assert.Equal(int64(0), i.HealthAfterCols[colYellow])
|
||||
assert.Equal(int64(0), i.HealthAfterCols[colRed])
|
||||
assert.Equal(int64(0), i.HealthAfterCols[colGrey])
|
||||
}
|
||||
|
||||
// Test-2: startHeal error on init
|
||||
minioHealMock = func(ctx context.Context, bucket, prefix string, healOpts madmin.HealOpts, clientToken string,
|
||||
forceStart, forceStop bool) (healStart madmin.HealStartSuccess, healTaskStatus madmin.HealTaskStatus, err error) {
|
||||
return healStart, mockHealTaskStatus, errors.New("error")
|
||||
}
|
||||
|
||||
if err := startHeal(ctx, mockWSConn, client, testOptions); assert.Error(err) {
|
||||
assert.Equal("error", err.Error())
|
||||
}
|
||||
|
||||
// Test-3: getHealOptionsFromReq return heal options from request
|
||||
u, _ := url.Parse("http://localhost/api/v1/heal/bucket1?prefix=file/&recursive=true&force-start=true&force-stop=true&remove=true&dry-run=true&scan=deep")
|
||||
req := &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts, err := getHealOptionsFromReq(req)
|
||||
if err != nil {
|
||||
t.Errorf("Failed on %s:, error occurred: %s", "getHealOptionsFromReq", err.Error())
|
||||
}
|
||||
expectedOptions := healOptions{
|
||||
BucketName: "bucket1",
|
||||
ForceStart: true,
|
||||
ForceStop: true,
|
||||
Prefix: "file/",
|
||||
HealOpts: madmin.HealOpts{
|
||||
Recursive: true,
|
||||
DryRun: true,
|
||||
ScanMode: madmin.HealDeepScan,
|
||||
},
|
||||
}
|
||||
assert.Equal(expectedOptions.BucketName, opts.BucketName)
|
||||
assert.Equal(expectedOptions.Prefix, opts.Prefix)
|
||||
assert.Equal(expectedOptions.Recursive, opts.Recursive)
|
||||
assert.Equal(expectedOptions.ForceStart, opts.ForceStart)
|
||||
assert.Equal(expectedOptions.DryRun, opts.DryRun)
|
||||
assert.Equal(expectedOptions.ScanMode, opts.ScanMode)
|
||||
// Test-3: getHealOptionsFromReq return error if boolean value not valid
|
||||
u, _ = url.Parse("http://localhost/api/v1/heal/bucket1?prefix=file/&recursive=nonbool&force-start=true&force-stop=true&remove=true&dry-run=true&scan=deep")
|
||||
req = &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts, err = getHealOptionsFromReq(req)
|
||||
if assert.Error(err) {
|
||||
assert.Equal("strconv.ParseBool: parsing \"nonbool\": invalid syntax", err.Error())
|
||||
}
|
||||
// Test-4: getHealOptionsFromReq return error if boolean value not valid
|
||||
u, _ = url.Parse("http://localhost/api/v1/heal/bucket1?prefix=file/&recursive=true&force-start=true&force-stop=true&remove=nonbool&dry-run=true&scan=deep")
|
||||
req = &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts, err = getHealOptionsFromReq(req)
|
||||
if assert.Error(err) {
|
||||
assert.Equal("strconv.ParseBool: parsing \"nonbool\": invalid syntax", err.Error())
|
||||
}
|
||||
// Test-5: getHealOptionsFromReq return error if boolean value not valid
|
||||
u, _ = url.Parse("http://localhost/api/v1/heal/bucket1?prefix=file/&recursive=true&force-start=nonbool&force-stop=true&remove=true&dry-run=true&scan=deep")
|
||||
req = &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts, err = getHealOptionsFromReq(req)
|
||||
if assert.Error(err) {
|
||||
assert.Equal("strconv.ParseBool: parsing \"nonbool\": invalid syntax", err.Error())
|
||||
}
|
||||
// Test-6: getHealOptionsFromReq return error if boolean value not valid
|
||||
u, _ = url.Parse("http://localhost/api/v1/heal/bucket1?prefix=file/&recursive=true&force-start=true&force-stop=nonbool&remove=true&dry-run=true&scan=deep")
|
||||
req = &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts, err = getHealOptionsFromReq(req)
|
||||
if assert.Error(err) {
|
||||
assert.Equal("strconv.ParseBool: parsing \"nonbool\": invalid syntax", err.Error())
|
||||
}
|
||||
// Test-7: getHealOptionsFromReq return error if boolean value not valid
|
||||
u, _ = url.Parse("http://localhost/api/v1/heal/bucket1?prefix=file/&recursive=true&force-start=true&force-stop=true&remove=true&dry-run=nonbool&scan=deep")
|
||||
req = &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts, err = getHealOptionsFromReq(req)
|
||||
if assert.Error(err) {
|
||||
assert.Equal("strconv.ParseBool: parsing \"nonbool\": invalid syntax", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,8 @@ type MinioAdmin interface {
|
||||
serviceTrace(ctx context.Context, allTrace, errTrace bool) <-chan madmin.ServiceTraceInfo
|
||||
getLogs(ctx context.Context, node string, lineCnt int, logKind string) <-chan madmin.LogInfo
|
||||
accountUsageInfo(ctx context.Context) (madmin.AccountUsageInfo, error)
|
||||
heal(ctx context.Context, bucket, prefix string, healOpts madmin.HealOpts, clientToken string,
|
||||
forceStart, forceStop bool) (healStart madmin.HealStartSuccess, healTaskStatus madmin.HealTaskStatus, err error)
|
||||
// Service Accounts
|
||||
addServiceAccount(ctx context.Context, policy *iampolicy.Policy) (mauth.Credentials, error)
|
||||
listServiceAccounts(ctx context.Context) (madmin.ListServiceAccountsResp, error)
|
||||
@@ -234,6 +236,11 @@ func (ac adminClient) accountUsageInfo(ctx context.Context) (madmin.AccountUsage
|
||||
return ac.client.AccountUsageInfo(ctx)
|
||||
}
|
||||
|
||||
func (ac adminClient) heal(ctx context.Context, bucket, prefix string, healOpts madmin.HealOpts, clientToken string,
|
||||
forceStart, forceStop bool) (healStart madmin.HealStartSuccess, healTaskStatus madmin.HealTaskStatus, err error) {
|
||||
return ac.client.Heal(ctx, bucket, prefix, healOpts, clientToken, forceStart, forceStop)
|
||||
}
|
||||
|
||||
func newMAdminClient(jwt string) (*madmin.AdminClient, error) {
|
||||
claims, err := auth.JWTAuthenticate(jwt)
|
||||
if err != nil {
|
||||
@@ -249,7 +256,9 @@ func newMAdminClient(jwt string) (*madmin.AdminClient, error) {
|
||||
// newAdminFromClaims creates a minio admin from Decrypted claims using Assume role credentials
|
||||
func newAdminFromClaims(claims *auth.DecryptedClaims) (*madmin.AdminClient, error) {
|
||||
tlsEnabled := getMinIOEndpointIsSecure()
|
||||
adminClient, err := madmin.NewWithOptions(getMinIOEndpoint(), &madmin.Options{
|
||||
endpoint := getMinIOEndpoint()
|
||||
|
||||
adminClient, err := madmin.NewWithOptions(endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(claims.AccessKeyID, claims.SecretAccessKey, claims.SessionToken),
|
||||
Secure: tlsEnabled,
|
||||
})
|
||||
|
||||
@@ -78,10 +78,10 @@ func startWatch(ctx context.Context, conn WSConn, wsc MCS3Client, options watchO
|
||||
}
|
||||
}
|
||||
|
||||
// getOptionsFromReq gets bucket name, events, prefix, suffix from a websocket
|
||||
// getWatchOptionsFromReq gets bucket name, events, prefix, suffix from a websocket
|
||||
// watch path if defined.
|
||||
// path come as : `/watch/bucket1` and query params come on request form
|
||||
func getOptionsFromReq(req *http.Request) watchOptions {
|
||||
func getWatchOptionsFromReq(req *http.Request) watchOptions {
|
||||
wOptions := watchOptions{}
|
||||
// Default Events if not defined
|
||||
wOptions.Events = []string{"put", "get", "delete"}
|
||||
|
||||
@@ -44,7 +44,7 @@ func TestWatch(t *testing.T) {
|
||||
mockWSConn := mockConn{}
|
||||
ctx := context.Background()
|
||||
|
||||
function := "startWatch(ctx, )"
|
||||
function := "startWatch()"
|
||||
|
||||
testStreamSize := 5
|
||||
testReceiver := make(chan []mc.EventInfo, testStreamSize)
|
||||
@@ -191,7 +191,7 @@ func TestWatch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test-6: getOptionsFromReq return parameters from path
|
||||
// Test-6: getWatchOptionsFromReq return parameters from path
|
||||
u, err := url.Parse("http://localhost/api/v1/watch/bucket1?prefix=&suffix=.jpg&events=put,get")
|
||||
if err != nil {
|
||||
t.Errorf("Failed on %s:, error occurred: %s", "url.Parse()", err.Error())
|
||||
@@ -199,7 +199,7 @@ func TestWatch(t *testing.T) {
|
||||
req := &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts := getOptionsFromReq(req)
|
||||
opts := getWatchOptionsFromReq(req)
|
||||
expectedOptions := watchOptions{
|
||||
BucketName: "bucket1",
|
||||
}
|
||||
@@ -211,7 +211,7 @@ func TestWatch(t *testing.T) {
|
||||
assert.Equal(expectedOptions.Suffix, opts.Suffix)
|
||||
assert.Equal(expectedOptions.Events, opts.Events)
|
||||
|
||||
// Test-7: getOptionsFromReq return default events if not defined
|
||||
// Test-7: getWatchOptionsFromReq return default events if not defined
|
||||
u, err = url.Parse("http://localhost/api/v1/watch/bucket1?prefix=&suffix=.jpg&events=")
|
||||
if err != nil {
|
||||
t.Errorf("Failed on %s:, error occurred: %s", "url.Parse()", err.Error())
|
||||
@@ -219,7 +219,7 @@ func TestWatch(t *testing.T) {
|
||||
req = &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts = getOptionsFromReq(req)
|
||||
opts = getWatchOptionsFromReq(req)
|
||||
expectedOptions = watchOptions{
|
||||
BucketName: "bucket1",
|
||||
}
|
||||
@@ -231,7 +231,7 @@ func TestWatch(t *testing.T) {
|
||||
assert.Equal(expectedOptions.Suffix, opts.Suffix)
|
||||
assert.Equal(expectedOptions.Events, opts.Events)
|
||||
|
||||
// Test-8: getOptionsFromReq return default events if not defined
|
||||
// Test-8: getWatchOptionsFromReq return default events if not defined
|
||||
u, err = url.Parse("http://localhost/api/v1/watch/bucket2?prefix=&suffix=")
|
||||
if err != nil {
|
||||
t.Errorf("Failed on %s:, error occurred: %s", "url.Parse()", err.Error())
|
||||
@@ -239,7 +239,7 @@ func TestWatch(t *testing.T) {
|
||||
req = &http.Request{
|
||||
URL: u,
|
||||
}
|
||||
opts = getOptionsFromReq(req)
|
||||
opts = getWatchOptionsFromReq(req)
|
||||
expectedOptions = watchOptions{
|
||||
BucketName: "bucket2",
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ type wsAdminClient struct {
|
||||
// MCSWebsocket interface of a Websocket Client
|
||||
type MCSWebsocket interface {
|
||||
watch(options watchOptions)
|
||||
heal(opts healOptions)
|
||||
}
|
||||
|
||||
type wsS3Client struct {
|
||||
@@ -125,29 +126,41 @@ func serveWS(w http.ResponseWriter, req *http.Request) {
|
||||
case wsPath == "/trace":
|
||||
wsAdminClient, err := newWebSocketAdminClient(conn, claims)
|
||||
if err != nil {
|
||||
errors.ServeError(w, req, err)
|
||||
closeWsConn(conn)
|
||||
return
|
||||
}
|
||||
go wsAdminClient.trace()
|
||||
case wsPath == "/console":
|
||||
wsAdminClient, err := newWebSocketAdminClient(conn, claims)
|
||||
if err != nil {
|
||||
errors.ServeError(w, req, err)
|
||||
closeWsConn(conn)
|
||||
return
|
||||
}
|
||||
go wsAdminClient.console()
|
||||
case strings.HasPrefix(wsPath, `/heal`):
|
||||
hOptions, err := getHealOptionsFromReq(req)
|
||||
if err != nil {
|
||||
log.Println("error getting heal options:", err)
|
||||
closeWsConn(conn)
|
||||
return
|
||||
}
|
||||
wsAdminClient, err := newWebSocketAdminClient(conn, claims)
|
||||
if err != nil {
|
||||
closeWsConn(conn)
|
||||
return
|
||||
}
|
||||
go wsAdminClient.heal(hOptions)
|
||||
case strings.HasPrefix(wsPath, `/watch`):
|
||||
wOptions := getOptionsFromReq(req)
|
||||
wOptions := getWatchOptionsFromReq(req)
|
||||
wsS3Client, err := newWebSocketS3Client(conn, *sessionID, wOptions.BucketName)
|
||||
if err != nil {
|
||||
errors.ServeError(w, req, err)
|
||||
closeWsConn(conn)
|
||||
return
|
||||
}
|
||||
go wsS3Client.watch(wOptions)
|
||||
default:
|
||||
// path not found
|
||||
conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
conn.Close()
|
||||
closeWsConn(conn)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,6 +240,12 @@ func wsReadClientCtx(conn WSConn) context.Context {
|
||||
return ctx
|
||||
}
|
||||
|
||||
// closeWsConn sends Close Message and closes the websocket connection
|
||||
func closeWsConn(conn *websocket.Conn) {
|
||||
conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// trace serves madmin.ServiceTraceInfo
|
||||
// on a Websocket connection.
|
||||
func (wsc *wsAdminClient) trace() {
|
||||
@@ -240,25 +259,8 @@ func (wsc *wsAdminClient) trace() {
|
||||
ctx := wsReadClientCtx(wsc.conn)
|
||||
|
||||
err := startTraceInfo(ctx, wsc.conn, wsc.client)
|
||||
// Send Connection Close Message indicating the Status Code
|
||||
// see https://tools.ietf.org/html/rfc6455#page-45
|
||||
if err != nil {
|
||||
log.Println("err:", err)
|
||||
// If connection exceeded read deadline send Close
|
||||
// Message Policy Violation code since we don't want
|
||||
// to let the receiver figure out the read deadline.
|
||||
// This is a generic code designed if there is a
|
||||
// need to hide specific details about the policy.
|
||||
if nErr, ok := err.(net.Error); ok && nErr.Timeout() {
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.ClosePolicyViolation, ""))
|
||||
return
|
||||
}
|
||||
// else, internal server error
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error()))
|
||||
return
|
||||
}
|
||||
// normal closure
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
|
||||
sendWsCloseMessage(wsc.conn, err)
|
||||
}
|
||||
|
||||
// console serves madmin.GetLogs
|
||||
@@ -274,24 +276,8 @@ func (wsc *wsAdminClient) console() {
|
||||
ctx := wsReadClientCtx(wsc.conn)
|
||||
|
||||
err := startConsoleLog(ctx, wsc.conn, wsc.client)
|
||||
// Send Connection Close Message indicating the Status Code
|
||||
// see https://tools.ietf.org/html/rfc6455#page-45
|
||||
if err != nil {
|
||||
// If connection exceeded read deadline send Close
|
||||
// Message Policy Violation code since we don't want
|
||||
// to let the receiver figure out the read deadline.
|
||||
// This is a generic code designed if there is a
|
||||
// need to hide specific details about the policy.
|
||||
if nErr, ok := err.(net.Error); ok && nErr.Timeout() {
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.ClosePolicyViolation, ""))
|
||||
return
|
||||
}
|
||||
// else, internal server error
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error()))
|
||||
return
|
||||
}
|
||||
// normal closure
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
|
||||
sendWsCloseMessage(wsc.conn, err)
|
||||
}
|
||||
|
||||
func (wsc *wsS3Client) watch(params watchOptions) {
|
||||
@@ -305,8 +291,28 @@ func (wsc *wsS3Client) watch(params watchOptions) {
|
||||
ctx := wsReadClientCtx(wsc.conn)
|
||||
|
||||
err := startWatch(ctx, wsc.conn, wsc.client, params)
|
||||
// Send Connection Close Message indicating the Status Code
|
||||
// see https://tools.ietf.org/html/rfc6455#page-45
|
||||
|
||||
sendWsCloseMessage(wsc.conn, err)
|
||||
}
|
||||
|
||||
func (wsc *wsAdminClient) heal(opts *healOptions) {
|
||||
defer func() {
|
||||
log.Println("heal stopped")
|
||||
// close connection after return
|
||||
wsc.conn.close()
|
||||
}()
|
||||
log.Println("heal started")
|
||||
|
||||
ctx := wsReadClientCtx(wsc.conn)
|
||||
|
||||
err := startHeal(ctx, wsc.conn, wsc.client, opts)
|
||||
|
||||
sendWsCloseMessage(wsc.conn, err)
|
||||
}
|
||||
|
||||
// sendWsCloseMessage sends Websocket Connection Close Message indicating the Status Code
|
||||
// see https://tools.ietf.org/html/rfc6455#page-45
|
||||
func sendWsCloseMessage(conn WSConn, err error) {
|
||||
if err != nil {
|
||||
// If connection exceeded read deadline send Close
|
||||
// Message Policy Violation code since we don't want
|
||||
@@ -314,13 +320,13 @@ func (wsc *wsS3Client) watch(params watchOptions) {
|
||||
// This is a generic code designed if there is a
|
||||
// need to hide specific details about the policy.
|
||||
if nErr, ok := err.(net.Error); ok && nErr.Timeout() {
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.ClosePolicyViolation, ""))
|
||||
conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.ClosePolicyViolation, ""))
|
||||
return
|
||||
}
|
||||
// else, internal server error
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error()))
|
||||
conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error()))
|
||||
return
|
||||
}
|
||||
// normal closure
|
||||
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user