Skip to content

数据库管理页面-添加mysql/pgsql/mongo实例选择下拉菜单-并添加pgsql/mongo创建数据库功能 #1401

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

数据库管理页面-添加mysql/pgsql/mongo实例选择下拉菜单-并添加pgsql/mongo创建数据库功能 #1401

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e967057
数据库管理页面添加pgsql/mysql/mongo实例过滤和数据库创建功能
Mar 1, 2022
509d97b
exec_cmd修改使用mongo url方式
Mar 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions sql/engines/mongo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,8 +260,8 @@ def exec_cmd(self, sql, db_name=None, slave_ok=''):
auth_db = self.instance.db_name or 'admin'
try:
if not sql.startswith('var host='): #在master节点执行的情况
cmd = "{mongo} --quiet -u {uname} -p '{password}' {host}:{port}/{auth_db} <<\\EOF\ndb=db.getSiblingDB(\"{db_name}\");{slave_ok}printjson({sql})\nEOF".format(
mongo=mongo, uname=self.user, password=self.password, host=self.host, port=self.port, db_name=db_name, sql=sql, auth_db=auth_db, slave_ok=slave_ok)
cmd = "{mongo} --quiet mongodb://{uname}:'{password}'@{host}:{port}/{auth_db} <<\\EOF\ndb=db.getSiblingDB(\"{db_name}\");{slave_ok}printjson({sql})\nEOF".format(
mongo=mongo, uname=self.user, password=self.password, host=self.host, port=self.port, db_name=db_name, sql=sql, auth_db=auth_db, slave_ok=slave_ok)
else:
cmd = "{mongo} --quiet -u {user} -p '{password}' {host}:{port}/{auth_db} <<\\EOF\nrs.slaveOk();{sql}\nEOF".format(
mongo=mongo, user=self.user, password=self.password, host=self.host, port=self.port, db_name=db_name, sql=sql, auth_db=auth_db)
Expand Down Expand Up @@ -320,13 +320,18 @@ def execute_workflow(self, workflow):
"""执行上线单，返回Review set"""
return self.execute(db_name=workflow.db_name, sql=workflow.sqlworkflowcontent.sql_content)

def execute(self, db_name=None, sql=''):
def execute(self, db_name=None, sql='', ddl=''):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

没有看出来ddl和非ddl的区别，可以去除ddl入参和下面的判断

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mongo.py ddl执行用的还是之前的方法，确实没有区别可以去掉，是为了打印ddl字符串加了一个判断。

"""mongo命令执行语句"""
self.get_master()
execute_result = ReviewSet(full_sql=sql)
sql = sql.strip()
# 以；切分语句，逐句执行
sp_sql = sql.split(";")
#ddl create database语句
if ddl.lower() == 'true':
sp_sql = sql.split(";")
logger.debug("ddl:" + str(sp_sql))
else:
# 以；切分语句，逐句执行
sp_sql = sql.split(";")
line = 0
for exec_sql in sp_sql:
if not exec_sql == '':
Expand Down
29 changes: 29 additions & 0 deletions sql/engines/pgsql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from . import EngineBase
from .models import ResultSet, ReviewSet, ReviewResult
from sql.utils.data_masking import simple_column_mask
from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

__author__ = 'hhyo、yyukai'

Expand Down Expand Up @@ -299,6 +300,34 @@ def execute_workflow(self, workflow, close_conn=True):
self.close()
return execute_result

def execute(self, db_name=None, sql='', ddl='', close_conn=True):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

execute方法就是用于执行有影响的语句，可以去除ddl入参和判断，统一设置autocommit

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pgsql执行create database需要设置conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)自动提交，和dml操作默认放到一个block事务提交有冲突，dml保证事务一致性还是比较合理。

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

是否是ddl直接在方法内判断比较好，可以优化一下

execute_result = ResultSet(full_sql=sql)
# 删除注释语句，切分语句，将切换CURRENT_SCHEMA语句增加到切分结果中
sql = sqlparse.format(sql, strip_comments=True)
split_sql = sqlparse.split(sql)
statement = None
try:
conn = self.get_connection(db_name=db_name)
# 设置事务隔离级别
if ddl.lower() == 'true':
conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
else:
pass
cursor = conn.cursor()
# 逐条执行切分语句，追加到执行结果中
for statement in split_sql:
statement = statement.rstrip(';')
with FuncTimer() as t:
cursor.execute(statement)
conn.commit()
except Exception as e:
logger.warning(f"PGSQL命令执行报错，语句：{statement or sql}， 错误信息：{traceback.format_exc()}")
execute_result.error = str(e)
finally:
if close_conn:
self.close()
return execute_result

def close(self):
if self.conn:
self.conn.close()
Expand Down
168 changes: 125 additions & 43 deletions sql/instance_database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,15 @@
@permission_required('sql.menu_database', raise_exception=True)
def databases(request):
"""获取实例数据库列表"""
db_type1 = request.POST.get('db_type1')
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

通过instance_id获取实例对象后就可以判断db类型了，不需要外部再传入这个参数

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

嗯看了源代码instance_id获取实例可以得到db_type，不过前端传的db_type1是为了在选择数据类型后在自动过滤该db_type1的实例列表，和返回db_type没有关系吧。

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个方法是获取实例的数据库列表，看了下不需要这个

instance_id = request.POST.get('instance_id')
saved = True if request.POST.get('saved') == 'true' else False # 平台是否保存

if not instance_id:
if not instance_id or not db_type1:
return JsonResponse({'status': 0, 'msg': '', 'data': []})

try:
instance = user_instances(request.user, db_type=['mysql']).get(id=instance_id)
instance = user_instances(request.user, db_type=['mysql','pgsql','mongo']).get(id=instance_id)
except Instance.DoesNotExist:
return JsonResponse({'status': 1, 'msg': '你所在组未关联该实例', 'data': []})

Expand All @@ -41,46 +42,117 @@ def databases(request):
db['saved'] = True
cnf_dbs[f"{db['db_name']}"] = db

# 获取所有数据库
sql_get_db = """SELECT SCHEMA_NAME,DEFAULT_CHARACTER_SET_NAME,DEFAULT_COLLATION_NAME
FROM information_schema.SCHEMATA
WHERE SCHEMA_NAME NOT IN ('information_schema', 'performance_schema', 'mysql', 'test', 'sys');"""
query_engine = get_engine(instance=instance)
query_result = query_engine.query('information_schema', sql_get_db, close_conn=False)
if not query_result.error:
dbs = query_result.rows
# 获取数据库关联用户信息
rows = []
for db in dbs:
db_name = db[0]
sql_get_bind_users = f"""select group_concat(distinct(GRANTEE)),TABLE_SCHEMA
from information_schema.SCHEMA_PRIVILEGES
where TABLE_SCHEMA='{db_name}'
group by TABLE_SCHEMA;"""
bind_users = query_engine.query('information_schema', sql_get_bind_users, close_conn=False).rows
row = {
'db_name': db_name,
'charset': db[1],
'collation': db[2],
'grantees': bind_users[0][0].split(',') if bind_users else [],
'saved': False
}
# 合并数据
if db_name in cnf_dbs.keys():
row = dict(row, **cnf_dbs[db_name])
rows.append(row)
# 过滤参数
if saved:
rows = [row for row in rows if row['saved']]

result = {'status': 0, 'msg': 'ok', 'rows': rows}
if db_type1 == "pgsql":
# 获取所有数据库
sql_get_db = """SELECT datname,datctype,datcollate,datacl
FROM pg_database
where datname not in ('postgres','template1','template0');"""
query_engine = get_engine(instance=instance)
query_result = query_engine.query('postgres', sql_get_db, close_conn=False)
if not query_result.error:
dbs = query_result.rows
# 获取数据库关联用户信息
rows = []
for db in dbs:
db_name = db[0]
row = {
'db_name': db_name,
'charset': db[1],
'collation': db[2],
'grantees': db[3],
'saved': False
}
# 合并数据
if db_name in cnf_dbs.keys():
row = dict(row, **cnf_dbs[db_name])
rows.append(row)
# 过滤参数
if saved:
rows = [row for row in rows if row['saved']]

result = {'status': 0, 'msg': 'ok', 'rows': rows}
else:
result = {'status': 1, 'msg': query_result.error}

# 关闭连接
query_engine.close()
return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True),
content_type='application/json')
elif db_type1 == "mongo":
# 获取所有数据库
query_engine = get_engine(instance=instance)
query_result = query_engine.get_all_databases()
if not query_result.error:
dbs = query_result.rows
# 获取数据库关联用户信息
rows = []
for db in dbs:
db_name = db
row = {
'db_name': db_name,
'charset': '',
'collation': '',
'grantees': '',
'saved': False
}
# 合并数据
if db_name in cnf_dbs.keys():
row = dict(row, **cnf_dbs[db_name])
rows.append(row)
# 过滤参数
if saved:
rows = [row for row in rows if row['saved']]

result = {'status': 0, 'msg': 'ok', 'rows': rows}
else:
result = {'status': 1, 'msg': query_result.error}

# 关闭连接
query_engine.close()
return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True),
content_type='application/json')
else:
result = {'status': 1, 'msg': query_result.error}
# 获取所有数据库
sql_get_db = """SELECT SCHEMA_NAME,DEFAULT_CHARACTER_SET_NAME,DEFAULT_COLLATION_NAME
FROM information_schema.SCHEMATA
WHERE SCHEMA_NAME NOT IN ('information_schema', 'performance_schema', 'mysql', 'test', 'sys');"""
query_engine = get_engine(instance=instance)
query_result = query_engine.query('information_schema', sql_get_db, close_conn=False)
if not query_result.error:
dbs = query_result.rows
# 获取数据库关联用户信息
rows = []
for db in dbs:
db_name = db[0]
sql_get_bind_users = f"""select group_concat(distinct(GRANTEE)),TABLE_SCHEMA
from information_schema.SCHEMA_PRIVILEGES
where TABLE_SCHEMA='{db_name}'
group by TABLE_SCHEMA;"""
bind_users = query_engine.query('information_schema', sql_get_bind_users, close_conn=False).rows
row = {
'db_name': db_name,
'charset': db[1],
'collation': db[2],
'grantees': bind_users[0][0].split(',') if bind_users else [],
'saved': False
}
# 合并数据
if db_name in cnf_dbs.keys():
row = dict(row, **cnf_dbs[db_name])
rows.append(row)
# 过滤参数
if saved:
rows = [row for row in rows if row['saved']]

result = {'status': 0, 'msg': 'ok', 'rows': rows}
else:
result = {'status': 1, 'msg': query_result.error}

# 关闭连接
query_engine.close()
return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True),
content_type='application/json')

# 关闭连接
query_engine.close()
return HttpResponse(json.dumps(result, cls=ExtendJSONEncoder, bigint_as_string=True),
content_type='application/json')


@permission_required('sql.menu_database', raise_exception=True)
Expand All @@ -90,12 +162,13 @@ def create(request):
db_name = request.POST.get('db_name')
owner = request.POST.get('owner', '')
remark = request.POST.get('remark', '')
db_type1 = request.POST.get('db_type1')

if not all([db_name]):
return JsonResponse({'status': 1, 'msg': '参数不完整，请确认后提交', 'data': []})

try:
instance = user_instances(request.user, db_type=['mysql']).get(id=instance_id)
instance = user_instances(request.user, db_type=['mysql','pgsql','mongo']).get(id=instance_id)
except Instance.DoesNotExist:
return JsonResponse({'status': 1, 'msg': '你所在组未关联该实例', 'data': []})

Expand All @@ -108,7 +181,16 @@ def create(request):
db_name = MySQLdb.escape_string(db_name).decode('utf-8')

engine = get_engine(instance=instance)
exec_result = engine.execute(db_name='information_schema', sql=f"create database {db_name};")

if db_type1 == "pgsql":
exec_result = engine.execute(db_name='postgres', sql=f"create database {db_name};", ddl='true')
elif db_type1 == "mongo":
password = remark.strip()
exec_sql=f"db.createUser("+"{user:"+f"'{db_name}',pwd:"+f"'{password}',roles:["+"{role:'dbAdmin',db:"+f"'{db_name}'"+"}],mechanisms:['SCRAM-SHA-1']});db.testcoll.insert({testid:'test01'});"
exec_result = engine.execute(db_name=db_name, sql=exec_sql, ddl='true')
else:
exec_result = engine.execute(db_name='information_schema', sql=f"create database {db_name};")

if exec_result.error:
return JsonResponse({'status': 1, 'msg': exec_result.error})
# 保存到数据库
Expand All @@ -134,7 +216,7 @@ def edit(request):
return JsonResponse({'status': 1, 'msg': '参数不完整，请确认后提交', 'data': []})

try:
instance = user_instances(request.user, db_type=['mysql']).get(id=instance_id)
instance = user_instances(request.user, db_type=['mysql','pgsql','mongo']).get(id=instance_id)
except Instance.DoesNotExist:
return JsonResponse({'status': 1, 'msg': '你所在组未关联该实例', 'data': []})

Expand Down
36 changes: 27 additions & 9 deletions sql/templates/database.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,15 @@
{% block content %}
<!-- 自定义操作按钮-->
<div id="toolbar" class="form-inline pull-left">
<div class="form-group">
<select id="db_type1" class="form-control selectpicker" name="db_type"
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

db_type

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

嗯，这个修改成db_type_f,表示数据库类型过滤

title="请选择数据库类型"
data-live-search="true">
<option value="mysql">mysql</option>
<option value="pgsql">pgsql</option>
<option value="mongo">mongo</option>
</select>
</div>
<div class="form-group ">
<select id=instance class="form-control selectpicker" name="instance_list"
title="请选择实例"
Expand Down Expand Up @@ -68,11 +77,11 @@ <h4 class="modal-title">创建数据库</h4>
</div>
</div>
<div class="form-group row">
<label for="remark" class="col-sm-3 col-form-label">备注</label>
<label for="remark" class="col-sm-3 col-form-label">备注/mongo密码</label>
<div class="col-sm-9">
<input type="text" id="remark" class="form-control"
autocomplete="off"
placeholder="请输入备注">
placeholder="请输入备注/mongo密码">
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

不特意描述为密码比较好，这里就是填写描述信息的地方

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

嗯，这个可以修改回来。
mongo create database ddl是自动创建用户和密码了，为了不修改模型层，所以复用了一个字段：）。

</div>
</div>
</div>
Expand Down Expand Up @@ -178,11 +187,12 @@ <h4 class="modal-title">编辑/录入数据库信息</h4>
//请求服务数据时所传参数
queryParams:
function (params) {
if ($("#instance").val()) {
if ($("#instance").val() && $("#db_type1").val()) {
return {
search: params.search,
instance_id: $("#instance").val(),
saved: $("#saved").val()
saved: $("#saved").val(),
db_type1: $("#db_type1").val()
}
}
},
Expand All @@ -200,11 +210,18 @@ <h4 class="modal-title">编辑/录入数据库信息</h4>
title: '授权账号',
field: 'grantees',
formatter: function (value, row, index) {
if ( $("#db_type1").val() == 'mysql' ) {
let grantee = '';
for (let i = 0; i < value.length; i++) {
grantee = grantee + value[i] + '</br>'
}
return grantee
}
else {
let grantee = '';
grantee = value
return grantee
}
}
}, {
title: '负责人',
Expand All @@ -214,7 +231,7 @@ <h4 class="modal-title">编辑/录入数据库信息</h4>
title: '负责人',
field: 'owner_display'
}, {
title: '备注',
title: '备注/mongo密码',
field: 'remark'
}, {
title: '操作',
Expand Down Expand Up @@ -261,7 +278,7 @@ <h4 class="modal-title">编辑/录入数据库信息</h4>

//创建数据库
function create_database() {
if (!$("#instance").val()) {
if (!$("#instance").val() || !$("#db_type1").val()) {
alert("请选择实例！");
} else {
$.ajax({
Expand All @@ -272,7 +289,8 @@ <h4 class="modal-title">编辑/录入数据库信息</h4>
instance_id: $("#instance").val(),
db_name: $("#db_name").val(),
owner: $("#owner").val(),
remark: $("#remark").val()
remark: $("#remark").val(),
db_type1: $("#db_type1").val()
},
complete: function () {
},
Expand Down Expand Up @@ -339,13 +357,13 @@ <h4 class="modal-title">编辑/录入数据库信息</h4>
$(document).ready(function () {
database_list();
//获取用户实例列表
$(function () {
$("#db_type1").change(function () {
$.ajax({
type: "get",
url: "/group/user_all_instances/",
dataType: "json",
data: {
db_type: ['mysql']
db_type: [$("#db_type1").val()]
},
complete: function () {
},
Expand Down